<commit_before>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

package de.charite.compbio.exomiser.config;

import de.charite.compbio.exomiser.parsers.ClinVarParser;
import de.charite.compbio.exomiser.parsers.DbSnpFrequencyParser;
import de.charite.compbio.exomiser.parsers.DiseaseInheritanceCache;
import de.charite.compbio.exomiser.parsers.EntrezParser;
import de.charite.compbio.exomiser.parsers.EspFrequencyParser;
import de.charite.compbio.exomiser.parsers.HPOOntologyFileParser;
import de.charite.compbio.exomiser.parsers.MetaDataParser;
import de.charite.compbio.exomiser.parsers.MimToGeneParser;
import de.charite.compbio.exomiser.parsers.MorbidMapParser;
import de.charite.compbio.exomiser.parsers.NSFP2SQLDumpParser;
import de.charite.compbio.exomiser.parsers.Omim2GeneParser;
import de.charite.compbio.exomiser.parsers.OmimResourceGroupParser;
import de.charite.compbio.exomiser.parsers.PhenoSeriesParser;
import de.charite.compbio.exomiser.parsers.ResourceGroupParser;
import de.charite.compbio.exomiser.parsers.StringParser;
import de.charite.compbio.exomiser.parsers.StringResourceGroupParser;
import de.charite.compbio.exomiser.parsers.VariantFrequencyResourceGroupParser;
import de.charite.compbio.exomiser.resources.Resource;
import java.util.LinkedHashSet;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * Java configuration for providing {@code Resource} definitions for the 
 * application to work on.
 * 
 * If you want to add another data source, this is where you define the resource
 * and the application will pick it up and incorporate the data into the database.
 * Obviously you'll have to write a parser and a flyway migration to get the data 
 * into the database.
 * 
 * @author <NAME> <<EMAIL>>
 */
@Configuration
public class ResourceConfig {
    
    private static final Logger logger = LoggerFactory.getLogger(ResourceConfig.class);
    
    private static final String UCSC_URL = "ftp://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/";
    
    private Set<Resource> resources;
    
    @Bean
    public Set<Resource> resources() {
        if (resources == null) {
            logger.info("Making new set of Resources");
            resources = new LinkedHashSet();
//            resources.add(exampleResource());
            resources.add(hpoResource());
            //OMIM group
            resources.add(omimMimToGeneResource());
            resources.add(omimMorbidMapResource());
            resources.add(hpoPhenotypeAnnotationsResource());
            //StringDB group
            resources.add(stringEntrezToSymResource());
            resources.add(stringProteinLinksResource());
            //Exome Walker
            resources.add(exomeWalkerPhenotypicSeriesResource());
            resources.add(exomeWalkerOmimToGeneResource());
            
            resources.add(clinVarResource());
            resources.add(metaDataResource());
            
            //these ones are biggies:
            resources.add(dbNsfpResource());
            //VariantFrequency group
            resources.add(dbSnpResource());
            resources.add(espResource());
            resources.add(hgResource());

            //UCSC resource files for making the ucsc.ser file - 
            //these are download only
            resources.add(ucscKnownGeneResource());
            resources.add(ucscKnownGeneXrefResource());
            resources.add(ucscKnownGeneMrnaResource());
            resources.add(ucscKnownToLocusLinkResource());       
        }
        
        return resources;
        
    }
    
    @Bean
    public Resource exampleResource() {
        logger.info("Making example resource");
        Resource resource = new Resource("Example");
        resource.setUrl("");
        resource.setRemoteFileName("");
        resource.setVersion("");
        resource.setExtractedFileName("");
        resource.setExtractionScheme("copy");
        //
        resource.setParserClass(null);
        resource.setParsedFileName("");
        //
        resource.setResourceGroupName("");
        resource.setResourceGroupParserClass(null);
        
        return resource;
    } 

    
    @Bean
    public Resource hpoResource() {
        logger.info("Making HPO resource");
        Resource resource = new Resource("HPO");
        resource.setUrl("http://compbio.charite.de/hudson/job/hpo/lastStableBuild/artifact/ontology/release/");
        resource.setRemoteFileName("hp.obo");
        resource.setVersion("");
        resource.setExtractedFileName("hpo.obo");
        resource.setExtractionScheme("copy");
        //TODO: this ought to return an actual class or instantiated object?
        resource.setParserClass(HPOOntologyFileParser.class);
        resource.setParsedFileName("hpo.pg");
        //
        resource.setResourceGroupName("");
        resource.setResourceGroupParserClass(null);
        
        return resource;
    } 
    
    @Bean
    public Resource dbNsfpResource() {
        logger.info("Making dbNSFP resource");
        Resource resource = new Resource("dbNSFP");
        // this is a 4.4GB file so might take a while
        // it is also about 25GB when uncompresssed
        resource.setUrl("http://dbnsfp.houstonbioinformatics.org/dbNSFPzip/");
        resource.setRemoteFileName("dbNSFPv2.4.zip");
        resource.setVersion("2.4");
        resource.setExtractedFileName("dbNSFPv2.4.zip");
        resource.setExtractionScheme("copy");
        //TODO: this ought to return an actual class or instantiated object?
        resource.setParserClass(NSFP2SQLDumpParser.class);
        resource.setParsedFileName("variant.pg");
        //
        resource.setResourceGroupName("");
        resource.setResourceGroupParserClass(null);
        
        return resource;
    } 
    
    @Bean
    public Resource dbSnpResource() {
        logger.info("Making dbSNP resource");
        Resource resource = new Resource("dbSNP");
        resource.setUrl("ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606/VCF/");
        resource.setRemoteFileName("00-All.vcf.gz");
        resource.setVersion("00");
        resource.setExtractedFileName("dbSNP_00-All.vcf");
        resource.setExtractionScheme("copy"); //can also do a gz 
        //TODO: this ought to return an actual class or instantiated object?
        resource.setParserClass(DbSnpFrequencyParser.class);
        resource.setParsedFileName("frequency.pg");
        //TODO: define groups seperately using DI
        resource.setResourceGroupName(VariantFrequencyResourceGroupParser.NAME);
        resource.setResourceGroupParserClass(VariantFrequencyResourceGroupParser.class);
        
        return resource;
    }
    
    @Bean
    public Resource espResource() {
        logger.info("Making ESP resource");
        Resource resource = new Resource("ESP");
        resource.setUrl("http://evs.gs.washington.edu/evs_bulk_data/");
        resource.setRemoteFileName("ESP6500SI-V2-SSA137.protein-hgvs-update.snps_indels.vcf.tar.gz");
        resource.setVersion("");
        resource.setExtractedFileName("ESP_snps_indels");
        resource.setExtractionScheme("tgz");
        //TODO: this ought to return an actual class or instantiated object?
        resource.setParserClass(EspFrequencyParser.class);
        resource.setParsedFileName("frequency.pg");
        //TODO: define groups seperately using DI
        resource.setResourceGroupName(VariantFrequencyResourceGroupParser.NAME);
        resource.setResourceGroupParserClass(VariantFrequencyResourceGroupParser.class);
        
        return resource;
    }
    
    @Bean<commit_msg>Added comments to resource groups.<commit_after>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

package de.charite.compbio.exomiser.config;

import de.charite.compbio.exomiser.parsers.ClinVarParser;
import de.charite.compbio.exomiser.parsers.DbSnpFrequencyParser;
import de.charite.compbio.exomiser.parsers.DiseaseInheritanceCache;
import de.charite.compbio.exomiser.parsers.EntrezParser;
import de.charite.compbio.exomiser.parsers.EspFrequencyParser;
import de.charite.compbio.exomiser.parsers.HPOOntologyFileParser;
import de.charite.compbio.exomiser.parsers.MetaDataParser;
import de.charite.compbio.exomiser.parsers.MimToGeneParser;
import de.charite.compbio.exomiser.parsers.MorbidMapParser;
import de.charite.compbio.exomiser.parsers.NSFP2SQLDumpParser;
import de.charite.compbio.exomiser.parsers.Omim2GeneParser;
import de.charite.compbio.exomiser.parsers.OmimResourceGroupParser;
import de.charite.compbio.exomiser.parsers.PhenoSeriesParser;
import de.charite.compbio.exomiser.parsers.ResourceGroupParser;
import de.charite.compbio.exomiser.parsers.StringParser;
import de.charite.compbio.exomiser.parsers.StringResourceGroupParser;
import de.charite.compbio.exomiser.parsers.VariantFrequencyResourceGroupParser;
import de.charite.compbio.exomiser.resources.Resource;
import java.util.LinkedHashSet;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * Java configuration for providing {@code Resource} definitions for the 
 * application to work on.
 * 
 * If you want to add another data source, this is where you define the resource
 * and the application will pick it up and incorporate the data into the database.
 * Obviously you'll have to write a parser and a flyway migration to get the data 
 * into the database.
 * 
 * @author <NAME> <<EMAIL>>
 */
@Configuration
public class ResourceConfig {
    
    private static final Logger logger = LoggerFactory.getLogger(ResourceConfig.class);
    
    private static final String UCSC_URL = "ftp://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/";
    
    private Set<Resource> resources;
    
    @Bean
    public Set<Resource> resources() {
        if (resources == null) {
            logger.info("Making new set of Resources");
            resources = new LinkedHashSet();
//            resources.add(exampleResource());
            resources.add(hpoResource());
            //OMIM group
            resources.add(omimMimToGeneResource());
            resources.add(omimMorbidMapResource());
            resources.add(hpoPhenotypeAnnotationsResource());
            //StringDB group
            resources.add(stringEntrezToSymResource());
            resources.add(stringProteinLinksResource());
            //Exome Walker
            resources.add(exomeWalkerPhenotypicSeriesResource());
            resources.add(exomeWalkerOmimToGeneResource());
            
            resources.add(clinVarResource());
            resources.add(metaDataResource());
            
            //these ones are biggies:
            resources.add(dbNsfpResource());
            //VariantFrequency group
            resources.add(dbSnpResource());
            resources.add(espResource());
            resources.add(hgResource());

            //UCSC resource files for making the ucsc.ser file - 
            //these are download only
            resources.add(ucscKnownGeneResource());
            resources.add(ucscKnownGeneXrefResource());
            resources.add(ucscKnownGeneMrnaResource());
            resources.add(ucscKnownToLocusLinkResource());       
        }
        
        return resources;
        
    }
    
    @Bean
    public Resource exampleResource() {
        logger.info("Making example resource");
        Resource resource = new Resource("Example");
        resource.setUrl("");
        resource.setRemoteFileName("");
        resource.setVersion("");
        resource.setExtractedFileName("");
        resource.setExtractionScheme("copy");
        //
        resource.setParserClass(null);
        resource.setParsedFileName("");
        //
        resource.setResourceGroupName("");
        resource.setResourceGroupParserClass(null);
        
        return resource;
    } 

    
    @Bean
    public Resource hpoResource() {
        logger.info("Making HPO resource");
        Resource resource = new Resource("HPO");
        resource.setUrl("http://compbio.charite.de/hudson/job/hpo/lastStableBuild/artifact/ontology/release/");
        resource.setRemoteFileName("hp.obo");
        resource.setVersion("");
        resource.setExtractedFileName("hpo.obo");
        resource.setExtractionScheme("copy");
        //parsing
        resource.setParserClass(HPOOntologyFileParser.class);
        resource.setParsedFileName("hpo.pg");
        //resource groups
        resource.setResourceGroupName("");
        resource.setResourceGroupParserClass(null);
        
        return resource;
    } 
    
    @Bean
    public Resource dbNsfpResource() {
        logger.info("Making dbNSFP resource");
        Resource resource = new Resource("dbNSFP");
        // this is a 4.4GB file so might take a while
        // it is also about 25GB when uncompresssed
        resource.setUrl("http://dbnsfp.houstonbioinformatics.org/dbNSFPzip/");
        resource.setRemoteFileName("dbNSFPv2.4.zip");
        resource.setVersion("2.4");
        resource.setExtractedFileName("dbNSFPv2.4.zip");
        resource.setExtractionScheme("copy");
        //parsing
        resource.setParserClass(NSFP2SQLDumpParser.class);
        resource.setParsedFileName("variant.pg");
        //
        resource.setResourceGroupName("");
        resource.setResourceGroupParserClass(null);
        
        return resource;
    } 
    
    @Bean
    public Resource dbSnpResource() {
        logger.info("Making dbSNP resource");
        Resource resource = new Resource("dbSNP");
        resource.setUrl("ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606/VCF/");
        resource.setRemoteFileName("00-All.vcf.gz");
        resource.setVersion("00");
        resource.setExtractedFileName("dbSNP_00-All.vcf");
        resource.setExtractionScheme("copy"); //can also do a gz 
        //parsing
        resource.setParserClass(DbSnpFrequencyParser.class);
        resource.setParsedFileName("frequency.pg");
        //resource groups
        resource.setResourceGroupName(VariantFrequencyResourceGroupParser.NAME);
        resource.setResourceGroupParserClass(VariantFrequencyResourceGroupParser.class);
        
        return resource;
    }
    
    @Bean
    public Resource espResource() {
        logger.info("Making ESP resource");
        Resource resource = new Resource("ESP");
        resource.setUrl("http://evs.gs.washington.edu/evs_bulk_data/");
        resource.setRemoteFileName("ESP6500SI-V2-SSA137.protein-hgvs-update.snps_indels.vcf.tar.gz");
        resource.setVersion("");
        resource.setExtractedFileName("ESP_snps_indels");
        resource.setExtractionScheme("tgz");
        //parsing
        resource.setParserClass(EspFrequencyParser.class);
        resource.setParsedFileName("frequency.pg");
        //resource groups
        resource.setResourceGroupName(VariantFrequencyResourceGroupParser.NAME);
        resource.setResourceGroupParserClass(VariantFrequencyResourceGroupParser.class);
        
        return resource;
    }
    
    @Bean
{'fraction_non_alphanumeric': 0.10373095791376194, 'fraction_numerical': 0.0045184611412341855, 'mean_word_length': 2.9817527627859164, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1894059', 'n_tokens_mistral': 4371, 'n_tokens_neox': 4200, 'n_words': 773}