<commit_before>/*******************************************************************************
 * Copyright 2015 EMBL - European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific
 * language governing permissions and limitations under the
 * License.
 *******************************************************************************/
package org.mousephenotype.cda.indexers;

import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.mousephenotype.cda.db.dao.DatasourceDAO;
import org.mousephenotype.cda.db.pojo.Datasource;
import org.mousephenotype.cda.db.pojo.Xref;
import org.mousephenotype.cda.indexers.exceptions.IndexerException;
import org.mousephenotype.cda.indexers.exceptions.ValidationException;
import org.mousephenotype.cda.indexers.utils.EmbryoRestGetter;
import org.mousephenotype.cda.indexers.utils.EmbryoStrain;
import org.mousephenotype.cda.indexers.utils.IndexerMap;
import org.mousephenotype.cda.solr.SolrUtils;
import org.mousephenotype.cda.solr.service.ImpressService;
import org.mousephenotype.cda.solr.service.dto.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.config.AutowireCapableBeanFactory;

import javax.annotation.Resource;
import javax.sql.DataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;

/**
 * Populate the MA core
 */
public class GeneIndexer extends AbstractIndexer {

    private static final Logger logger = LoggerFactory.getLogger(GeneIndexer.class);
    private Connection komp2DbConnection;

    @Autowired
    @Qualifier("komp2DataSource")
    DataSource komp2DataSource;

    @Autowired
    @Qualifier("alleleReadOnlyIndexing")
    SolrServer alleleCore;

    @Autowired
    @Qualifier("geneIndexing")
    SolrServer geneCore;

    @Autowired
    @Qualifier("mpReadOnlyIndexing")
    SolrServer mpCore;

    @Autowired
    @Qualifier("sangerImagesReadOnlyIndexing")
    SolrServer imagesCore;

    @Autowired
	DatasourceDAO datasourceDAO;

    @Autowired
    ImpressService ims;

    private Map<String, List<Map<String, String>>> phenotypeSummaryGeneAccessionsToPipelineInfo = new HashMap<>();
    private Map<String, Map<String, String>> genomicFeatureCoordinates = new HashMap<>();
    private Map<String, List<Xref>> genomicFeatureXrefs = new HashMap<>();
    private Map<String, List<SangerImageDTO>> sangerImages = new HashMap<>();
    private Map<String, List<MpDTO>> mgiAccessionToMP = new HashMap<>();
    Map<String, List<EmbryoStrain>> embryoRestData=null;


    EmbryoRestGetter embryoGetter;

    @Resource(name = "globalConfiguration")
    private Map<String, String> config;

    public GeneIndexer() {

    }

    @Override
    public void validateBuild() throws IndexerException {
        Long numFound = getDocumentCount(geneCore);

        if (numFound <= MINIMUM_DOCUMENT_COUNT)
            throw new IndexerException(new ValidationException("Actual gene document count is " + numFound + "."));

        if (numFound != documentCount)
            logger.warn("WARNING: Added " + documentCount + " gene documents but SOLR reports " + numFound + " documents.");
        else
            logger.info("validateBuild(): Indexed " + documentCount + " gene documents.");
    }

    @Override
    public void initialise(String[] args) throws IndexerException {

        super.initialise(args);
        applicationContext.getAutowireCapableBeanFactory().autowireBeanProperties(this, AutowireCapableBeanFactory.AUTOWIRE_BY_TYPE, true);

        try {

            komp2DbConnection = komp2DataSource.getConnection();

        } catch (SQLException sqle) {
            logger.error("Caught SQL Exception initialising database connections: {}", sqle.getMessage());
            throw new IndexerException(sqle);
        }

    }

    @Override
    public void run() throws IndexerException {


    	Datasource ensembl = datasourceDAO.getDatasourceByShortName("Ensembl");
		Datasource vega = datasourceDAO.getDatasourceByShortName("VEGA");
		Datasource ncbi = datasourceDAO.getDatasourceByShortName("EntrezGene");
		Datasource ccds = datasourceDAO.getDatasourceByShortName("cCDS");

        long startTime = System.currentTimeMillis();
        try {
            logger.info("Starting Gene Indexer...");

            initialiseSupportingBeans();

            int count = 0;
            List<AlleleDTO> alleles = IndexerMap.getAlleles(alleleCore);
            logger.info("alleles size=" + alleles.size());

            geneCore.deleteByQuery("*:*");

            for (AlleleDTO allele : alleles) {
                //System.out.println("allele="+allele.getMarkerSymbol());
                GeneDTO gene = new GeneDTO();
                gene.setMgiAccessionId(allele.getMgiAccessionId());
                gene.setDataType(allele.getDataType());
                gene.setMarkerType(allele.getMarkerType());
                gene.setMarkerSymbol(allele.getMarkerSymbol());
                gene.setMarkerSymbolLowercase(allele.getMarkerSymbol());
                gene.setMarkerSynonym(allele.getMarkerSynonym());
                gene.setMarkerName(allele.getMarkerName());
                gene.setHumanGeneSymbol(allele.getHumanGeneSymbol());
                gene.setEnsemblGeneIds(allele.getEnsemblGeneIds());
                gene.setLatestEsCellStatus(allele.getLatestEsCellStatus());
                gene.setImitsPhenotypeStarted(allele.getImitsPhenotypeStarted());
                gene.setImitsPhenotypeComplete(allele.getImitsPhenotypeComplete());
                gene.setImitsPhenotypeStatus(allele.getImitsPhenotypeStatus());
                gene.setLatestMouseStatus(allele.getLatestMouseStatus());
                gene.setLatestProjectStatus(allele.getLatestProjectStatus());
                gene.setStatus(allele.getStatus());
                gene.setLatestPhenotypeStatus(allele.getLatestPhenotypeStatus());
                gene.setLegacy_phenotype_status(allele.getLegacyPhenotypeStatus());
                gene.setLatestProductionCentre(allele.getLatestProductionCentre());
                gene.setLatestPhenotypingCentre(allele.getLatestPhenotypingCentre());
                gene.setAlleleName(allele.getAlleleName());
                gene.setEsCellStatus(allele.getEsCellStatus());
                gene.setMouseStatus(allele.getMouseStatus());
                gene.setPhenotypeStatus(allele.getPhenotypeStatus());
                gene.setProductionCentre(allele.getProductionCentre());
                gene.setPhenotypingCentre(allele.getPhenotypingCentre());
                gene.setType(allele.getType());
                gene.setDiseaseSource(allele.getDiseaseSource());
                gene.setDiseaseId(allele.getDiseaseId());
                gene.setDiseaseTerm(allele.getDiseaseTerm());
                gene.setDiseaseAlts(allele.getDiseaseAlts());
                gene.setDiseaseClasses(allele.getDiseaseClasses());
                gene.setHumanCurated(allele.getHumanCurated());
                gene.setMouseCurated(allele.getMouseCurated());
                gene.setMgiPredicted(allele.getMgiPredicted());
                gene.setImpcPredicted(allele.getImpcPredicted());
                gene.setMgiPredicted(allele.getMgiPredicted());
                gene.setMgiPredictedKnonwGene(allele.getMgiPredictedKnownGene());
                gene.setImpcNovelPredictedInLocus(allele.getImpcNovelPredictedInLocus());
                gene.setDiseaseHumanPhenotypes(allele.getDiseaseHumanPhenotypes());

                // GO stuff
                gene.setGoTermIds(allele.getGoTermIds());
                gene.setGoTermNames(allele.getGoTermNames());
               // gene.getGoTermDefs().addAll(allele.getGoTermDefs());
                gene.setGoTermEvids(allele.getGoTermEvids());<commit_msg>Uncommented adding of GO definition in the gene core.<commit_after>/*******************************************************************************
 * Copyright 2015 EMBL - European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific
 * language governing permissions and limitations under the
 * License.
 *******************************************************************************/
package org.mousephenotype.cda.indexers;

import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.mousephenotype.cda.db.dao.DatasourceDAO;
import org.mousephenotype.cda.db.pojo.Datasource;
import org.mousephenotype.cda.db.pojo.Xref;
import org.mousephenotype.cda.indexers.exceptions.IndexerException;
import org.mousephenotype.cda.indexers.exceptions.ValidationException;
import org.mousephenotype.cda.indexers.utils.EmbryoRestGetter;
import org.mousephenotype.cda.indexers.utils.EmbryoStrain;
import org.mousephenotype.cda.indexers.utils.IndexerMap;
import org.mousephenotype.cda.solr.SolrUtils;
import org.mousephenotype.cda.solr.service.ImpressService;
import org.mousephenotype.cda.solr.service.dto.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.config.AutowireCapableBeanFactory;

import javax.annotation.Resource;
import javax.sql.DataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;

/**
 * Populate the MA core
 */
public class GeneIndexer extends AbstractIndexer {

    private static final Logger logger = LoggerFactory.getLogger(GeneIndexer.class);
    private Connection komp2DbConnection;

    @Autowired
    @Qualifier("komp2DataSource")
    DataSource komp2DataSource;

    @Autowired
    @Qualifier("alleleReadOnlyIndexing")
    SolrServer alleleCore;

    @Autowired
    @Qualifier("geneIndexing")
    SolrServer geneCore;

    @Autowired
    @Qualifier("mpReadOnlyIndexing")
    SolrServer mpCore;

    @Autowired
    @Qualifier("sangerImagesReadOnlyIndexing")
    SolrServer imagesCore;

    @Autowired
	DatasourceDAO datasourceDAO;

    @Autowired
    ImpressService ims;

    private Map<String, List<Map<String, String>>> phenotypeSummaryGeneAccessionsToPipelineInfo = new HashMap<>();
    private Map<String, Map<String, String>> genomicFeatureCoordinates = new HashMap<>();
    private Map<String, List<Xref>> genomicFeatureXrefs = new HashMap<>();
    private Map<String, List<SangerImageDTO>> sangerImages = new HashMap<>();
    private Map<String, List<MpDTO>> mgiAccessionToMP = new HashMap<>();
    Map<String, List<EmbryoStrain>> embryoRestData=null;


    EmbryoRestGetter embryoGetter;

    @Resource(name = "globalConfiguration")
    private Map<String, String> config;

    public GeneIndexer() {

    }

    @Override
    public void validateBuild() throws IndexerException {
        Long numFound = getDocumentCount(geneCore);

        if (numFound <= MINIMUM_DOCUMENT_COUNT)
            throw new IndexerException(new ValidationException("Actual gene document count is " + numFound + "."));

        if (numFound != documentCount)
            logger.warn("WARNING: Added " + documentCount + " gene documents but SOLR reports " + numFound + " documents.");
        else
            logger.info("validateBuild(): Indexed " + documentCount + " gene documents.");
    }

    @Override
    public void initialise(String[] args) throws IndexerException {

        super.initialise(args);
        applicationContext.getAutowireCapableBeanFactory().autowireBeanProperties(this, AutowireCapableBeanFactory.AUTOWIRE_BY_TYPE, true);

        try {

            komp2DbConnection = komp2DataSource.getConnection();

        } catch (SQLException sqle) {
            logger.error("Caught SQL Exception initialising database connections: {}", sqle.getMessage());
            throw new IndexerException(sqle);
        }

    }

    @Override
    public void run() throws IndexerException {


    	Datasource ensembl = datasourceDAO.getDatasourceByShortName("Ensembl");
		Datasource vega = datasourceDAO.getDatasourceByShortName("VEGA");
		Datasource ncbi = datasourceDAO.getDatasourceByShortName("EntrezGene");
		Datasource ccds = datasourceDAO.getDatasourceByShortName("cCDS");

        long startTime = System.currentTimeMillis();
        try {
            logger.info("Starting Gene Indexer...");

            initialiseSupportingBeans();

            int count = 0;
            List<AlleleDTO> alleles = IndexerMap.getAlleles(alleleCore);
            logger.info("alleles size=" + alleles.size());

            geneCore.deleteByQuery("*:*");

            for (AlleleDTO allele : alleles) {
                //System.out.println("allele="+allele.getMarkerSymbol());
                GeneDTO gene = new GeneDTO();
                gene.setMgiAccessionId(allele.getMgiAccessionId());
                gene.setDataType(allele.getDataType());
                gene.setMarkerType(allele.getMarkerType());
                gene.setMarkerSymbol(allele.getMarkerSymbol());
                gene.setMarkerSymbolLowercase(allele.getMarkerSymbol());
                gene.setMarkerSynonym(allele.getMarkerSynonym());
                gene.setMarkerName(allele.getMarkerName());
                gene.setHumanGeneSymbol(allele.getHumanGeneSymbol());
                gene.setEnsemblGeneIds(allele.getEnsemblGeneIds());
                gene.setLatestEsCellStatus(allele.getLatestEsCellStatus());
                gene.setImitsPhenotypeStarted(allele.getImitsPhenotypeStarted());
                gene.setImitsPhenotypeComplete(allele.getImitsPhenotypeComplete());
                gene.setImitsPhenotypeStatus(allele.getImitsPhenotypeStatus());
                gene.setLatestMouseStatus(allele.getLatestMouseStatus());
                gene.setLatestProjectStatus(allele.getLatestProjectStatus());
                gene.setStatus(allele.getStatus());
                gene.setLatestPhenotypeStatus(allele.getLatestPhenotypeStatus());
                gene.setLegacy_phenotype_status(allele.getLegacyPhenotypeStatus());
                gene.setLatestProductionCentre(allele.getLatestProductionCentre());
                gene.setLatestPhenotypingCentre(allele.getLatestPhenotypingCentre());
                gene.setAlleleName(allele.getAlleleName());
                gene.setEsCellStatus(allele.getEsCellStatus());
                gene.setMouseStatus(allele.getMouseStatus());
                gene.setPhenotypeStatus(allele.getPhenotypeStatus());
                gene.setProductionCentre(allele.getProductionCentre());
                gene.setPhenotypingCentre(allele.getPhenotypingCentre());
                gene.setType(allele.getType());
                gene.setDiseaseSource(allele.getDiseaseSource());
                gene.setDiseaseId(allele.getDiseaseId());
                gene.setDiseaseTerm(allele.getDiseaseTerm());
                gene.setDiseaseAlts(allele.getDiseaseAlts());
                gene.setDiseaseClasses(allele.getDiseaseClasses());
                gene.setHumanCurated(allele.getHumanCurated());
                gene.setMouseCurated(allele.getMouseCurated());
                gene.setMgiPredicted(allele.getMgiPredicted());
                gene.setImpcPredicted(allele.getImpcPredicted());
                gene.setMgiPredicted(allele.getMgiPredicted());
                gene.setMgiPredictedKnonwGene(allele.getMgiPredictedKnownGene());
                gene.setImpcNovelPredictedInLocus(allele.getImpcNovelPredictedInLocus());
                gene.setDiseaseHumanPhenotypes(allele.getDiseaseHumanPhenotypes());

                // GO stuff
                gene.setGoTermIds(allele.getGoTermIds());
                gene.setGoTermNames(allele.getGoTermNames());
                gene.getGoTermDefs().addAll(allele.getGoTermDefs());
                gene.setGoTermEvids(allele.getGoTermEvids());
{'fraction_non_alphanumeric': 0.11642472488878483, 'fraction_numerical': 0.001990166237415125, 'mean_word_length': 3.4353582554517135, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '140480', 'n_tokens_mistral': 5029, 'n_tokens_neox': 4727, 'n_words': 783}