<commit_before>package org.opencb.cellbase.app.cli;

import com.beust.jcommander.ParameterException;
import org.apache.commons.lang.StringUtils;
import org.opencb.cellbase.core.CellBaseConfiguration.SpeciesProperties.Species;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

/**
 * Created by imedina on 03/02/15.
 */
public class DownloadCommandExecutor extends CommandExecutor {

    private File ensemblScriptsFolder;
    private CliOptionsParser.DownloadCommandOptions downloadCommandOptions;

    private static final String[] variationFiles = {"variation.txt.gz", "variation_feature.txt.gz",
            "transcript_variation.txt.gz", "variation_synonym.txt.gz", "seq_region.txt.gz", "source.txt.gz",
            "attrib.txt.gz", "attrib_type.txt.gz", "seq_region.txt.gz", "structural_variation_feature.txt.gz",
            "study.txt.gz", "phenotype.txt.gz", "phenotype_feature.txt.gz", "phenotype_feature_attrib.txt.gz",
            "motif_feature_variation.txt.gz", "genotype_code.txt.gz", "allele_code.txt.gz",
            "population_genotype.txt.gz", "population.txt.gz", "allele.txt.gz"};

    private static final String[] regulationFiles = {"AnnotatedFeatures.gff.gz", "MotifFeatures.gff.gz",
            "RegulatoryFeatures_MultiCell.gff.gz"};

    private String ensemblVersion;
    private String ensemblRelease;

    public DownloadCommandExecutor(CliOptionsParser.DownloadCommandOptions downloadCommandOptions) {
        super(downloadCommandOptions.commonOptions.logLevel, downloadCommandOptions.commonOptions.verbose,
                downloadCommandOptions.commonOptions.conf);

        this.downloadCommandOptions = downloadCommandOptions;
        this.ensemblScriptsFolder = new File(System.getProperty("basedir") + "/bin/ensembl-scripts/");
    }


    /**
     * Execute specific 'download' command options
     */
    public void execute() {
        try {
            checkParameters();
            Path outputDir = Paths.get(downloadCommandOptions.output);
            makeDir(outputDir);

            // We need to get the Species object from the CLI name
            // This can be the scientific or common name, or the ID
            Species speciesToDownload = null;
            for (Species species: configuration.getAllSpecies()) {
                if (downloadCommandOptions.species.equalsIgnoreCase(species.getScientificName())
                        || downloadCommandOptions.species.equalsIgnoreCase(species.getCommonName())
                        || downloadCommandOptions.species.equalsIgnoreCase(species.getId())) {
                    speciesToDownload = species;
                    break;
                }
            }

            // If everything is right we launch the download
            if(speciesToDownload != null) {
                processSpecies(speciesToDownload, outputDir);
            }else {
                logger.error("Species '{}' not valid", downloadCommandOptions.species);
            }
        } catch (ParameterException e) {
            logger.error("Error in 'download' command line: " + e.getMessage());
        } catch (IOException | InterruptedException e) {
            logger.error("Error downloading '" + downloadCommandOptions.species + "' files: " + e.getMessage());
        }

    }

    private void checkParameters() {
        if (!downloadCommandOptions.genome && !downloadCommandOptions.gene && !downloadCommandOptions.variation
                && !downloadCommandOptions.regulation && !downloadCommandOptions.protein
                && !downloadCommandOptions.conservation && !downloadCommandOptions.clinical) {
            throw new ParameterException("At least one 'download' option must be selected: sequence, gene, variation, regulation, protein");
        }
    }

    private void processSpecies(Species sp, Path outputDir) throws IOException, InterruptedException {
        logger.info("Processing species " + sp.getScientificName());

        // We need to find which is the correct Ensembl host URL.
        // This can different depending on if is a vertebrate species.
        String ensemblHostUrl;
        if (configuration.getSpecies().getVertebrates().contains(sp)) {
            ensemblHostUrl = configuration.getDownload().getEnsembl().getUrl().getHost();
        } else {
            ensemblHostUrl = configuration.getDownload().getEnsemblGenomes().getUrl().getHost();
        }

        // Getting the assembly.
        // By default the first assembly in the configuration.json
        Species.Assembly assembly = null;
        if(downloadCommandOptions.assembly == null || downloadCommandOptions.assembly.isEmpty()) {
            assembly = sp.getAssemblies().get(0);
        }else {
            for (Species.Assembly assembly1 : sp.getAssemblies()) {
                if(downloadCommandOptions.assembly.equalsIgnoreCase(assembly1.getName())) {
                    assembly = assembly1;
                    break;
                }
            }
        }

        // Checking that the species and assembly are correct
        if(ensemblHostUrl == null || assembly == null) {
            logger.error("Something is not correct, check the species '{}' or the assembly '{}'",
                    downloadCommandOptions.species, downloadCommandOptions.assembly);
            return;
        }

        // Output folder creation
        String spShortName = sp.getScientificName().toLowerCase().replaceAll("\\.", "").replaceAll("\\)", "")
                .replaceAll("[-(/]", " ").replaceAll("\\s+", "_");
        String spAssembly = assembly.getName().toLowerCase();
        Path spFolder = outputDir.resolve(spShortName + "_" + spAssembly);
        makeDir(spFolder);

        ensemblVersion = assembly.getEnsemblVersion();
        ensemblRelease = "release-" + ensemblVersion.split("_")[0];

        // download sequence, gene, variation, regulation and protein
        if (downloadCommandOptions.genome && speciesHasInfoToDownload(sp, "genome")) {
            downloadReferenceGenome(sp, spShortName, assembly.getName(), spFolder, ensemblHostUrl);
        }
        if (downloadCommandOptions.gene && speciesHasInfoToDownload(sp, "gene")) {
            downloadEnsemblGene(sp, spShortName, spFolder, ensemblHostUrl);
        }
        if (downloadCommandOptions.variation && speciesHasInfoToDownload(sp, "variation")) {
            downloadVariation(sp, spShortName, assembly.getName(), spFolder, ensemblHostUrl);
        }
        if (downloadCommandOptions.regulation && speciesHasInfoToDownload(sp, "regulation")) {
            downloadRegulation(sp, spShortName, assembly.getName(), spFolder, ensemblHostUrl);
        }
        if (downloadCommandOptions.protein && speciesHasInfoToDownload(sp, "protein")) {
            downloadProtein(sp, spShortName, assembly.getName(), spFolder);
        }
        if (downloadCommandOptions.conservation && speciesHasInfoToDownload(sp, "conservation")) {
            downloadConservation(sp, assembly.getName(), spFolder);
        }
        if (downloadCommandOptions.clinical && speciesHasInfoToDownload(sp, "clinical")) {
            downloadProtein(sp, spShortName, assembly.getName(), spFolder);
        }
    }


    private boolean speciesHasInfoToDownload(Species sp, String info) {
        boolean hasInfo = true;
        if (sp.getData() == null || !sp.getData().contains(info)) {
            logger.warn("Specie " + sp.getScientificName() + " has no " + info + " information available to download");
            hasInfo = false;
        }
        return hasInfo;
    }

    private String getPhylo(Species sp) {
        if (configuration.getSpecies().getVertebrates().contains(sp)) {<commit_msg>Download CLI learns parameter 'all', this will download all the species data in the configuration.json<commit_after>package org.opencb.cellbase.app.cli;

import com.beust.jcommander.ParameterException;
import org.apache.commons.lang.StringUtils;
import org.opencb.cellbase.core.CellBaseConfiguration.SpeciesProperties.Species;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

/**
 * Created by imedina on 03/02/15.
 */
public class DownloadCommandExecutor extends CommandExecutor {

    private File ensemblScriptsFolder;
    private CliOptionsParser.DownloadCommandOptions downloadCommandOptions;

    private static final String[] variationFiles = {"variation.txt.gz", "variation_feature.txt.gz",
            "transcript_variation.txt.gz", "variation_synonym.txt.gz", "seq_region.txt.gz", "source.txt.gz",
            "attrib.txt.gz", "attrib_type.txt.gz", "seq_region.txt.gz", "structural_variation_feature.txt.gz",
            "study.txt.gz", "phenotype.txt.gz", "phenotype_feature.txt.gz", "phenotype_feature_attrib.txt.gz",
            "motif_feature_variation.txt.gz", "genotype_code.txt.gz", "allele_code.txt.gz",
            "population_genotype.txt.gz", "population.txt.gz", "allele.txt.gz"};

    private static final String[] regulationFiles = {"AnnotatedFeatures.gff.gz", "MotifFeatures.gff.gz",
            "RegulatoryFeatures_MultiCell.gff.gz"};

    private String ensemblVersion;
    private String ensemblRelease;

    public DownloadCommandExecutor(CliOptionsParser.DownloadCommandOptions downloadCommandOptions) {
        super(downloadCommandOptions.commonOptions.logLevel, downloadCommandOptions.commonOptions.verbose,
                downloadCommandOptions.commonOptions.conf);

        this.downloadCommandOptions = downloadCommandOptions;
        this.ensemblScriptsFolder = new File(System.getProperty("basedir") + "/bin/ensembl-scripts/");
    }


    /**
     * Execute specific 'download' command options
     */
    public void execute() {
        try {
            checkParameters();
            Path outputDir = Paths.get(downloadCommandOptions.output);
            makeDir(outputDir);

            // We need to get the Species object from the CLI name
            // This can be the scientific or common name, or the ID
            Species speciesToDownload = null;
            for (Species species: configuration.getAllSpecies()) {
                if (downloadCommandOptions.species.equalsIgnoreCase(species.getScientificName())
                        || downloadCommandOptions.species.equalsIgnoreCase(species.getCommonName())
                        || downloadCommandOptions.species.equalsIgnoreCase(species.getId())) {
                    speciesToDownload = species;
                    break;
                }
            }

            // If everything is right we launch the download
            if(speciesToDownload != null) {
                processSpecies(speciesToDownload, outputDir);
            }else {
                logger.error("Species '{}' not valid", downloadCommandOptions.species);
            }
        } catch (ParameterException e) {
            logger.error("Error in 'download' command line: " + e.getMessage());
        } catch (IOException | InterruptedException e) {
            logger.error("Error downloading '" + downloadCommandOptions.species + "' files: " + e.getMessage());
        }

    }

    private void checkParameters() {
        if (!downloadCommandOptions.genome && !downloadCommandOptions.gene && !downloadCommandOptions.variation
                && !downloadCommandOptions.regulation && !downloadCommandOptions.protein
                && !downloadCommandOptions.conservation && !downloadCommandOptions.clinical
                && !downloadCommandOptions.all) {
            throw new ParameterException("At least one 'download' option must be selected: sequence, gene, variation, regulation, protein");
        }
    }

    private void processSpecies(Species sp, Path outputDir) throws IOException, InterruptedException {
        logger.info("Processing species " + sp.getScientificName());

        // We need to find which is the correct Ensembl host URL.
        // This can different depending on if is a vertebrate species.
        String ensemblHostUrl;
        if (configuration.getSpecies().getVertebrates().contains(sp)) {
            ensemblHostUrl = configuration.getDownload().getEnsembl().getUrl().getHost();
        } else {
            ensemblHostUrl = configuration.getDownload().getEnsemblGenomes().getUrl().getHost();
        }

        // Getting the assembly.
        // By default the first assembly in the configuration.json
        Species.Assembly assembly = null;
        if(downloadCommandOptions.assembly == null || downloadCommandOptions.assembly.isEmpty()) {
            assembly = sp.getAssemblies().get(0);
        }else {
            for (Species.Assembly assembly1 : sp.getAssemblies()) {
                if(downloadCommandOptions.assembly.equalsIgnoreCase(assembly1.getName())) {
                    assembly = assembly1;
                    break;
                }
            }
        }

        // Checking that the species and assembly are correct
        if(ensemblHostUrl == null || assembly == null) {
            logger.error("Something is not correct, check the species '{}' or the assembly '{}'",
                    downloadCommandOptions.species, downloadCommandOptions.assembly);
            return;
        }

        // Output folder creation
        String spShortName = sp.getScientificName().toLowerCase().replaceAll("\\.", "").replaceAll("\\)", "")
                .replaceAll("[-(/]", " ").replaceAll("\\s+", "_");
        String spAssembly = assembly.getName().toLowerCase();
        Path spFolder = outputDir.resolve(spShortName + "_" + spAssembly);
        makeDir(spFolder);

        ensemblVersion = assembly.getEnsemblVersion();
        ensemblRelease = "release-" + ensemblVersion.split("_")[0];

        // download sequence, gene, variation, regulation and protein
        if ((downloadCommandOptions.genome && speciesHasInfoToDownload(sp, "genome")) || downloadCommandOptions.all) {
            downloadReferenceGenome(sp, spShortName, assembly.getName(), spFolder, ensemblHostUrl);
        }
        if ((downloadCommandOptions.gene && speciesHasInfoToDownload(sp, "gene")) || downloadCommandOptions.all) {
            downloadEnsemblGene(sp, spShortName, spFolder, ensemblHostUrl);
        }
        if ((downloadCommandOptions.variation && speciesHasInfoToDownload(sp, "variation")) || downloadCommandOptions.all) {
            downloadVariation(sp, spShortName, assembly.getName(), spFolder, ensemblHostUrl);
        }
        if ((downloadCommandOptions.regulation && speciesHasInfoToDownload(sp, "regulation")) || downloadCommandOptions.all) {
            downloadRegulation(sp, spShortName, assembly.getName(), spFolder, ensemblHostUrl);
        }
        if ((downloadCommandOptions.protein && speciesHasInfoToDownload(sp, "protein")) || downloadCommandOptions.all) {
            downloadProtein(sp, spShortName, assembly.getName(), spFolder);
        }
        if ((downloadCommandOptions.conservation && speciesHasInfoToDownload(sp, "conservation")) || downloadCommandOptions.all) {
            downloadConservation(sp, assembly.getName(), spFolder);
        }
        if ((downloadCommandOptions.clinical && speciesHasInfoToDownload(sp, "clinical")) || downloadCommandOptions.all) {
            downloadProtein(sp, spShortName, assembly.getName(), spFolder);
        }
    }


    private boolean speciesHasInfoToDownload(Species sp, String info) {
        boolean hasInfo = true;
        if (sp.getData() == null || !sp.getData().contains(info)) {
            logger.warn("Specie " + sp.getScientificName() + " has no " + info + " information available to download");
            hasInfo = false;
        }
        return hasInfo;
    }

    private String getPhylo(Species sp) {
        if (configuration.getSpecies().getVertebrates().contains(sp)) {
{'fraction_non_alphanumeric': 0.11164987405541561, 'fraction_numerical': 0.0013853904282115869, 'mean_word_length': 2.815713599231139, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 138, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5015858', 'n_tokens_mistral': 4164, 'n_tokens_neox': 3980, 'n_words': 846}