<commit_before>package uk.ac.ebi.spot.goci.service;

import org.semanticweb.HermiT.Reasoner;
import org.semanticweb.owlapi.io.RDFXMLOntologyFormat;
import org.semanticweb.owlapi.model.*;
import org.semanticweb.owlapi.reasoner.*;
import org.semanticweb.owlapi.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import uk.ac.ebi.spot.goci.dao.JDBCSingleNucleotidePolymorphismDAO;
import uk.ac.ebi.spot.goci.dao.JDBCStudyDAO;
import uk.ac.ebi.spot.goci.dao.JDBCTraitAssociationDAO;
import uk.ac.ebi.spot.goci.exception.OWLConversionException;
import uk.ac.ebi.spot.goci.exception.ObjectMappingException;
import uk.ac.ebi.spot.goci.exception.OntologyTermException;
import uk.ac.ebi.spot.goci.lang.FilterProperties;
import uk.ac.ebi.spot.goci.lang.OntologyConfiguration;
import uk.ac.ebi.spot.goci.ui.model.SingleNucleotidePolymorphism;
import uk.ac.ebi.spot.goci.ui.model.Study;
import uk.ac.ebi.spot.goci.ui.model.TraitAssociation;
import uk.ac.ebi.spot.goci.utils.OntologyUtils;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;

/**
 * Javadocs go here!
 *
 * @author <NAME> Date 26/01/12
 */
public class DefaultGWASOWLPublisher implements GWASOWLPublisher {
    private OntologyConfiguration configuration;
    private int studiesLimit = -1;

    private JDBCStudyDAO studyDAO;
    private JDBCTraitAssociationDAO traitAssociationDAO;
    private JDBCSingleNucleotidePolymorphismDAO singleNucleotidePolymorphismDAO;
    private GWASOWLConverter converter;

    private Logger log = LoggerFactory.getLogger(getClass());

    protected Logger getLog() {
        return log;
    }

    public void setConfiguration(OntologyConfiguration configuration) {
        this.configuration = configuration;
    }

    public int getStudiesLimit() {
        return studiesLimit;
    }

    public void setStudiesLimit(int studiesLimit) {
        this.studiesLimit = studiesLimit;
    }

    public JDBCStudyDAO getStudyDAO() {
        return studyDAO;
    }

    public void setStudyDAO(JDBCStudyDAO studyDAO) {
        this.studyDAO = studyDAO;
    }

    public JDBCTraitAssociationDAO getTraitAssociationDAO() {
        return traitAssociationDAO;
    }

    public void setTraitAssociationDAO(JDBCTraitAssociationDAO traitAssociationDAO) {
        this.traitAssociationDAO = traitAssociationDAO;
    }

    public JDBCSingleNucleotidePolymorphismDAO getSingleNucleotidePolymorphismDAO() {
        return singleNucleotidePolymorphismDAO;
    }

    public void setSingleNucleotidePolymorphismDAO(JDBCSingleNucleotidePolymorphismDAO singleNucleotidePolymorphismDAO) {
        this.singleNucleotidePolymorphismDAO = singleNucleotidePolymorphismDAO;
    }

    public GWASOWLConverter getConverter() {
        return converter;
    }

    public void setConverter(GWASOWLConverter converter) {
        this.converter = converter;
    }

    public OWLOntologyManager getManager() {
        return configuration.getOWLOntologyManager();
    }

    public OWLOntology publishGWASData() throws OWLConversionException {
        // create new ontology
        OWLOntology conversion = getConverter().createConversionOntology();

        // grab all studies from the DAO
        getLog().debug("Fetching studies that require conversion to OWL using JDBCStudyDAO...");
        Collection<Study> studies = getStudyDAO().retrieveAllStudies();
        getLog().debug("Query complete, got " + studies.size() + " studies");
//            validateGWASData(studies);

        // if studies limit is not set, convert all data, else filter to first n studies and associated data
        if (getStudiesLimit() == -1 &&
                FilterProperties.getDateFilter() == null &&
                FilterProperties.getPvalueFilter() == null) {
            // grab all other data from the DAO
            getLog().debug("Fetching traits that require conversion to OWL using JDBCTraitAssociationDAO...");
            Collection<TraitAssociation> traitAssociations = getTraitAssociationDAO().retrieveAllTraitAssociations();
            getLog().debug("Fetching SNPs that require conversion to OWL using JDBCSingleNucleotidePolymorphismDAO...");
            Collection<SingleNucleotidePolymorphism> snps = getSingleNucleotidePolymorphismDAO().retrieveAllSNPs();
            getLog().debug("All data fetched");

            // convert this data, starting with SNPs (no dependencies) and working up to studies
            getLog().debug("Starting conversion to OWL...");
            getLog().debug("Converting SNPs...");
            getConverter().addSNPsToOntology(snps, conversion);
            getLog().debug("Converting Trait Associations...");
            getConverter().addAssociationsToOntology(traitAssociations, conversion);
            getLog().debug("Converting Studies...");
            getConverter().addStudiesToOntology(studies, conversion);
            getLog().debug("All conversion done!");

            return conversion;
        }
        else {
            return filterAndPublishGWASData(conversion, studies);
        }
    }

    private OWLOntology filterAndPublishGWASData(OWLOntology conversion, Collection<Study> studies)
            throws OWLConversionException {
        Collection<Study> filteredStudies = new ArrayList<Study>();
        Collection<TraitAssociation> filteredTraitAssociations = new ArrayList<TraitAssociation>();
        Collection<SingleNucleotidePolymorphism> filteredSNPs = new ArrayList<SingleNucleotidePolymorphism>();

        int count = 0;
        int studyLimit = getStudiesLimit() == -1 ? Integer.MAX_VALUE : getStudiesLimit();
        Iterator<Study> studyIterator = studies.iterator();
        while (count < studyLimit && studyIterator.hasNext()) {
            Study nextStudy = studyIterator.next();
            filteredStudies.add(nextStudy);
            for (TraitAssociation nextTA : nextStudy.getIdentifiedAssociations()) {
                filteredTraitAssociations.add(nextTA);
                try {
                    filteredSNPs.add(nextTA.getAssociatedSNP());
                }
                catch (ObjectMappingException e) {
                    // we can safely ignore this, a warning will be issued when we add each trait association
                }
            }
            count++;
        }

        // convert this data, starting with SNPs (no dependencies) and working up to studies
        getLog().debug("Starting conversion to OWL...");
        getLog().debug("Converting " + filteredSNPs.size() + " filtered SNPs...");
        getConverter().addSNPsToOntology(filteredSNPs, conversion);
        getLog().debug("Converting " + filteredTraitAssociations.size() + " filtered Trait Associations...");
        getConverter().addAssociationsToOntology(filteredTraitAssociations, conversion);
        getLog().debug("Converting " + filteredStudies.size() + " filtered Studies...");
        getConverter().addStudiesToOntology(filteredStudies, conversion);
        getLog().debug("All conversion done!");

        return conversion;
    }

    public OWLReasoner publishGWASDataInferredView(OWLOntology ontology) throws OWLConversionException {
        try {
            getLog().debug("Loading any missing imports...");
            OntologyUtils.loadImports(ontology.getOWLOntologyManager(), ontology);
            StringBuilder loadedOntologies = new StringBuilder();
            int n = 1;
            for (OWLOntology o : ontology.getOWLOntologyManager().getOntologies()) {
                loadedOntologies.append("\t")
                        .append(n++)
                        .append(") ")
                        .append(o.getOntologyID().getOntologyIRI())
                        .append("\n");
            }
            getLog().debug("Imports collected: the following ontologies have been loaded in this session:\n" +
                                   loadedOntologies.toString());
            getLog().info("Classifying ontology from " + ontology.getOntologyID().getOntologyIRI());

            getLog().debug("Creating reasoner... ");
            OWLReasonerFactory factory = new Reasoner.ReasonerFactory();
            ConsoleProgressMonitor progressMonitor = new ConsoleProgressMonitor();
            OWLReasonerConfiguration config = new SimpleConfiguration(progressMonitor);
            OWLReasoner reasoner = factory.createReasoner(ontology, config);

            getLog().debug("Precomputing inferences...");
            reasoner.precomputeInferences();

            getLog().debug("Checking ontology consistency...");
            reasoner.isConsistent();

            getLog().debug("Checking for unsatisfiable classes...");
            if (reasoner.getUnsatisfiableClasses().getEntitiesMinusBottom().size() > 0) {
                throw new OWLConversionException("Once classified, unsatisfiable classes were detected");
            }
            else {
                getLog().info("Reasoning complete! ");
                return reasoner;
            }
        }
        catch (UnloadableImportException e) {
            throw new OWLConversionException("Failed to load imports", e);
        }
    }

    public void saveGWASData(OWLOntology ontology, File outputFile) throws OWLConversionException {
        try {
            getLog().info("Saving GWAS catalog data...");
//            OWLXMLOntologyFormat format = new OWLXMLOntologyFormat();
            OWLOntologyFormat format = new RDFXMLOntologyFormat();
            getManager().saveOntology(ontology,
                                      format,
                                      IRI.create(outputFile));
            getLog().info("GWAS catalog data saved ok");
            getLog().info("Resulting ontology contains " + ontology.getAxiomCount() + " axioms " +
                                  "and is saved at " + outputFile.getAbsolutePath());
        }
        catch (OWLOntologyStorageException e) {
            throw new OWLConversionException("Failed to save GWAS data", e);
        }
    }

    public void saveGWASDataInferredView(OWLReasoner reasoner, File outputFile) throws OWLConversionException {
        try {
            // create new ontology to hold inferred axioms
            OWLOntology inferredOntology = getConverter().createConversionOntology();

            getLog().info("Saving inferred view...");
            List<InferredAxiomGenerator<? extends OWLAxiom>> gens =
                    new ArrayList<InferredAxiomGenerator<? extends OWLAxiom>>();
            // we require all inferred stuff except for disjoints...
            gens.add(new InferredClassAssertionAxiomGenerator());
            gens.add(new InferredDataPropertyCharacteristicAxiomGenerator());
            gens.add(new InferredEquivalentClassAxiomGenerator());
            gens.add(new InferredEquivalentDataPropertiesAxiomGenerator());
            gens.add(new InferredEquivalentObjectPropertyAxiomGenerator());
            gens.add(new InferredInverseObjectPropertiesAxiomGenerator());
            gens.add(new InferredObjectPropertyCharacteristicAxiomGenerator());
            gens.add(new InferredPropertyAssertionGenerator());
            gens.add(new InferredSubClassAxiomGenerator());
            gens.add(new InferredSubDataPropertyAxiomGenerator());
            gens.add(new InferredSubObjectPropertyAxiomGenerator());

            // now create the target ontology and save
            OWLOntologyManager inferredManager = inferredOntology.getOWLOntologyManager();
            InferredOntologyGenerator iog = new InferredOntologyGenerator(reasoner, gens);
            iog.fillOntology(inferredManager, inferredOntology);
            inferredManager.saveOntology(inferredOntology, IRI.create(outputFile));
            getLog().info("Inferred view saved ok");
        }
        catch (OWLOntologyStorageException e) {
            throw new OWLConversionException("Failed to save GWAS data (inferred view)", e);
        }
    }

    /**
     * Validates the data obtained from the GWAS catalog (prior to converting to OWL)
     *
     * @param studies the set of studies to validate
     */
    protected void validateGWASData(Collection<Study> studies) {
        // now check a random assortment of 5 studies for trait associations, abandoning broken ones
        int count = 0;
        int noAssocCount = 0;
        int termMismatches = 0;
        for (Study study : studies) {
            try {
                Collection<TraitAssociation> associations = study.getIdentifiedAssociations();
                getLog().debug("Study (PubMed ID '" + study.getPubMedID() + "') had " + associations.size() +
                                       " associations");
                if (associations.size() > 0) {
                    for (TraitAssociation association : associations) {
                        getLog().debug(
                                "    Association: SNP '" + association.getAssociatedSNP().getRSID() +
                                        "' <-> Trait '" +
                                        association.getAssociatedTrait().toString() + "'");
                    }
                    count++;
                }
                else {
                    noAssocCount++;
                }
            }
            catch (ObjectMappingException e) {
                if (e instanceof OntologyTermException) {
                    termMismatches++;
                    getLog().error("EFO term mapping failed: " + e.getMessage());
                }
                else {
                    getLog().warn("Excluding Study (PubMed ID '" + study.getPubMedID() + "'), " + e.getMessage());
                }
            }
        }
        int eligCount = studies.size() - noAssocCount;
        int correctCount = count + termMismatches;
        getLog().info("\n\nREPORT:\n" +
                              eligCount + "/" + studies.size() +
                              " declared associations and therefore could usefully be mapped.\n" +
                              (eligCount - count - termMismatches) + "/" + eligCount +
                              " failed due to data integrity concerns.\n" +
                              count + "/" + correctCount +
                              " studies could be completely mapped after passing all checks.\n" +
                              termMismatches + "/" + correctCount +
                              " failed due to missing or duplicated terms in EFO");
    }
}<commit_msg>update publisher to use new schema<commit_after>package uk.ac.ebi.spot.goci.service;

import org.semanticweb.HermiT.Reasoner;
import org.semanticweb.owlapi.io.RDFXMLOntologyFormat;
import org.semanticweb.owlapi.model.*;
import org.semanticweb.owlapi.reasoner.*;
import org.semanticweb.owlapi.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import uk.ac.ebi.spot.goci.exception.OWLConversionException;
import uk.ac.ebi.spot.goci.model.EfoTrait;
import uk.ac.ebi.spot.goci.model.Locus;
import uk.ac.ebi.spot.goci.model.RiskAllele;
import uk.ac.ebi.spot.goci.repository.AssociationRepository;
import uk.ac.ebi.spot.goci.repository.SingleNucleotidePolymorphismRepository;
import uk.ac.ebi.spot.goci.repository.StudyRepository;
import uk.ac.ebi.spot.goci.utils.FilterProperties;
import uk.ac.ebi.spot.goci.owl.OntologyLoader;
import uk.ac.ebi.spot.goci.model.SingleNucleotidePolymorphism;
import uk.ac.ebi.spot.goci.model.Study;
import uk.ac.ebi.spot.goci.model.Association;
//import uk.ac.ebi.spot.goci.utils.OntologyUtils;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;

/**
 * Javadocs go here!
 *
 * @author <NAME> Date 26/01/12
 */
@Service
public class DefaultGwasOwlPublisher implements GwasOwlPublisher {
    private OntologyLoader ontologyLoader;
    private int studiesLimit = -1;

    private StudyRepository studyRepository;
    private StudyService studyService;

    private AssociationRepository associationRepository;
    private AssociationService associationService;

    private SingleNucleotidePolymorphismRepository singleNucleotidePolymorphismRepository;
    private SingleNucleotidePolymorphismService singleNucleotidePolymorphismService;

    private GwasOwlConverter converter;




    @Autowired
//    public DefaultGwasOwlPublisher (StudyRepository studyRepository, AssociationRepository associationRepository, SingleNucleotidePolymorphismRepository snpRepository, GwasOwlConverter converter){
    public DefaultGwasOwlPublisher (StudyService studyService, AssociationService associationService, SingleNucleotidePolymorphismService singleNucleotidePolymorphismService, GwasOwlConverter converter, OntologyLoader ontologyLoader){
        this.studyService = studyService;
        this.associationService = associationService;
        this.singleNucleotidePolymorphismService = singleNucleotidePolymorphismService;
        this.converter = converter;
        this.ontologyLoader = ontologyLoader;
    }

    private Logger log = LoggerFactory.getLogger(getClass());

    protected Logger getLog() {
        return log;
    }

    public void setOntologyLoader(OntologyLoader ontologyLoader) {
        this.ontologyLoader = ontologyLoader;
    }

    public int getStudiesLimit() {
        return studiesLimit;
    }

    public void setStudiesLimit(int studiesLimit) {
        this.studiesLimit = studiesLimit;
    }

    public StudyRepository getStudyRepository() {
        return studyRepository;
    }

    public StudyService getStudyService(){
        return studyService;
    }


    public AssociationRepository getAssociationRepository() {
        return associationRepository;
    }
    public AssociationService getAssociationService() {
        return associationService;
    }



    public SingleNucleotidePolymorphismRepository getSingleNucleotidePolymorphismRepository() {
        return singleNucleotidePolymorphismRepository;
    }
    public SingleNucleotidePolymorphismService getSingleNucleotidePolymorphismService() {
        return singleNucleotidePolymorphismService;
    }



    public GwasOwlConverter getConverter() {
        return converter;
    }



    public OWLOntologyManager getManager() {
        return ontologyLoader.getOntology().getOWLOntologyManager();
    }

    public OWLOntology publishGWASData() throws OWLConversionException {
        // create new ontology
        OWLOntology conversion = getConverter().createConversionOntology();

        // grab all studies from the DAO
        getLog().debug("Fetching studies that require conversion to OWL using StudyRepository...");

        //TODO : this is where I changed from Repository to Service
//        Collection<Study> studies = getStudyRepository().findAll();
        Collection<Study> studies = getStudyService().findReallyAll();

        //TODO : check with Tony probably better to do it at the Repository/Service level
        Iterator<Study> iterator = studies.iterator();
        while(iterator.hasNext()){
            Study study = iterator.next();
            //Remove study which have no diseaseTrait.
            if(study.getDiseaseTrait() == null) {
                iterator.remove();
            }else if( study.getHousekeeping().getPublishDate() == null) {
                iterator.remove();
            }
//            }else {
//
//                //Remove study which have no associations where pvalue is not null.
//                Collection<Association> associations = study.getAssociations();
//                Iterator<Association> associationIterator = associations.iterator();
//                int associationCount = 0;
//                while (associationIterator.hasNext()) {
//                    Association association = associationIterator.next();
//
//                    if (association.getPvalueExponent() != null && association.getPvalueMantissa() != null) {
//                        associationCount++;
//                    }
//                }
//                if (associationCount == 0) {
//                    iterator.remove();
//                }
//            }
        }
//        for(Study study : studies){
//            if(study.getDiseaseTrait() == null){
//                studies.remove(study);
//            }
//        }

        getLog().debug("Query complete, got " + studies.size() + " studies");
//            validateGWASData(studies);

        // if studies limit is not set, convert all data, else filter to first n studies and associated data
        if (getStudiesLimit() == -1 &&
                FilterProperties.getDateFilter() == null &&
                FilterProperties.getPvalueFilter() == null) {
            // grab all other data from the DAO
            getLog().debug("Fetching traits that require conversion to OWL using AssociationRepository...");
            Collection<Association> traitAssociations = getAssociationService().findReallyAll();
            //todo check with Tony how to do that in a better way from service or repository (how to not get associations linked to study with no trait.
            Iterator<Association> associationIterator = traitAssociations.iterator();
            while(associationIterator.hasNext()){
                Association association = associationIterator.next();
                System.out.println("3 pvalue is "  + association.getPvalueMantissa()+"e"+association.getPvalueExponent() );
                if(association.getStudy().getDiseaseTrait() == null) {
                    associationIterator.remove();
//                }else if (association.getPvalueExponent() == null || association.getPvalueMantissa() == null){
//                    //Remove association if it's got no pvalue
//                    associationIterator.remove();

                }else if(association.getStudy().getHousekeeping().getPublishDate() == null){
                    associationIterator.remove();

                }
            }
            getLog().debug("Fetching SNPs that require conversion to OWL using SingleNucleotidePolymorphismRepository...");
            Collection<SingleNucleotidePolymorphism> snps = getSingleNucleotidePolymorphismService().findAll();
            getLog().debug("All data fetched");

            // convert this data, starting with SNPs (no dependencies) and working up to studies
            getLog().debug("Starting conversion to OWL...");
            getLog().debug("Converting SNPs...");
            getConverter().addSNPsToOntology(snps, conversion);
            getLog().debug("Converting Trait Associations...");
            getConverter().addAssociationsToOntology(traitAssociations, conversion);
            getLog().debug("Converting Studies...");
            getConverter().addStudiesToOntology(studies, conversion);
            getLog().debug("All conversion done!");

            return conversion;
        }
        else {
            return filterAndPublishGWASData(conversion, studies);
        }
    }

    private OWLOntology filterAndPublishGWASData(OWLOntology conversion, Collection<Study> studies)
            throws OWLConversionException {

        //TODO : check with tony : Discard studies which are not yet associated with a trait.
        Iterator<Study> iterator = studies.iterator();
        while(iterator.hasNext()){
            Study study = iterator.next();
            if(study.getDiseaseTrait() == null) {
                iterator.remove();
            }else if( study.getHousekeeping().getPublishDate() == null) {
                iterator.remove();
            }
//            }else {
//
//                //Remove study which have no associations where pvalue is not null.
//                Collection<Association> associations = study.getAssociations();
//                Iterator<Association> associationIterator = associations.iterator();
//                int associationCount = 0;
//                while (associationIterator.hasNext()) {
//                    Association association = associationIterator.next();
//
//                    if (association.getPvalueExponent() != null && association.getPvalueMantissa() != null) {
//                        associationCount++;
//                    }
//                }
//                if (associationCount == 0) {
//                    iterator.remove();
//                }
//            }
        }


        Collection<Study> filteredStudies = new ArrayList<Study>();
        Collection<Association> filteredTraitAssociations = new ArrayList<Association>();
        Collection<SingleNucleotidePolymorphism> filteredSNPs = new ArrayList<SingleNucleotidePolymorphism>();

        int count = 0;
        int studyLimit = getStudiesLimit() == -1 ? Integer.MAX_VALUE : getStudiesLimit();
        Iterator<Study> studyIterator = studies.iterator();
        while (count < studyLimit && studyIterator.hasNext()) {
            Study nextStudy = studyIterator.next();
                for (Association nextTA : nextStudy.getAssociations()) {
//                    System.out.println("nextTA.getStudy().getDiseaseTrait()" + nextTA.getStudy().getDiseaseTrait());
                    //TODO : check if ok, filtering out association with no pvalue, make sure that we don't end up with studies with no association
//                    if(nextTA.getPvalueExponent()!= null && nextTA.getPvalueMantissa() != null) {
                        System.out.println("2 pvalue is "  + nextTA.getPvalueMantissa()+"e"+nextTA.getPvalueExponent() );
                        filteredTraitAssociations.add(nextTA);
                        for (Locus locus : nextTA.getLoci()) {
                            for (RiskAllele riskAllele : locus.getStrongestRiskAlleles()) {
                                filteredSNPs.add(riskAllele.getSnp());
                            }
                        }
                        //  try {
                        //                    filteredSNPs.add(nextTA.getAssociatedSNP());
                        //}
                        //                catch (ObjectMappingException e) {
                        //                    // we can safely ignore this, a warning will be issued when we add each trait association
                        //                }
//                    }
                }
                //TODO : as we filter out the association if they don't have a pvalue, we make sure here that we don't
                // add study which would have not association left.
//                if(filteredTraitAssociations.size() != 0){
                    filteredStudies.add(nextStudy);
//                }
                count++;
        }
        // convert this data, starting with SNPs (no dependencies) and working up to studies
        getLog().debug("Starting conversion to OWL...");
        getLog().debug("Converting " + filteredSNPs.size() + " filtered SNPs...");
        getConverter().addSNPsToOntology(filteredSNPs, conversion);
        getLog().debug("Converting " + filteredTraitAssociations.size() + " filtered Trait Associations...");
        getConverter().addAssociationsToOntology(filteredTraitAssociations, conversion);
        getLog().debug("Converting " + filteredStudies.size() + " filtered Studies...");
        getConverter().addStudiesToOntology(filteredStudies, conversion);
        getLog().debug("All conversion done!");

        return conversion;
    }

    public OWLReasoner publishGWASDataInferredView(OWLOntology ontology) throws OWLConversionException {
//        try {
            getLog().debug("Loading any missing imports...");
//            OntologyUtils.loadImports(ontology.getOWLOntologyManager(), ontology);
            StringBuilder loadedOntologies = new StringBuilder();
            int n = 1;
            for (OWLOntology o : ontology.getOWLOntologyManager().getOntologies()) {
                loadedOntologies.append("\t")
                        .append(n++)
                        .append(") ")
                        .append(o.getOntologyID().getOntologyIRI())
                        .append("\n");
            }
            getLog().debug("Imports collected: the following ontologies have been loaded in this session:\n" +
                                   loadedOntologies.toString());
            getLog().info("Classifying ontology from " + ontology.getOntologyID().getOntologyIRI());

            getLog().debug("Creating reasoner... ");
            OWLReasonerFactory factory = new Reasoner.ReasonerFactory();
            ConsoleProgressMonitor progressMonitor = new ConsoleProgressMonitor();
            OWLReasonerConfiguration config = new SimpleConfiguration(progressMonitor);
            OWLReasoner reasoner = factory.createReasoner(ontology, config);

            getLog().debug("Precomputing inferences...");
            reasoner.precomputeInferences();

            getLog().debug("Checking ontology consistency...");
            reasoner.isConsistent();

            getLog().debug("Checking for unsatisfiable classes...");
            if (reasoner.getUnsatisfiableClasses().getEntitiesMinusBottom().size() > 0) {
                throw new OWLConversionException("Once classified, unsatisfiable classes were detected");
            }
            else {
                getLog().info("Reasoning complete! ");
                return reasoner;
            }
//        }
//        catch (UnloadableImportException e) {
//            throw new OWLConversionException("Failed to load imports", e);
//        }
    }

    public void saveGWASData(OWLOntology ontology, File outputFile) throws OWLConversionException {
        try {
            getLog().info("Saving GWAS catalog data...");
//            OWLXMLOntologyFormat format = new OWLXMLOntologyFormat();
            OWLOntologyFormat format = new RDFXMLOntologyFormat();
            getManager().saveOntology(ontology,
                                      format,
                                      IRI.create(outputFile));
            getLog().info("GWAS catalog data saved ok");
            getLog().info("Resulting ontology contains " + ontology.getAxiomCount() + " axioms " +
                                  "and is saved at " + outputFile.getAbsolutePath());
        }
        catch (OWLOntologyStorageException e) {
            throw new OWLConversionException("Failed to save GWAS data", e);
        }
    }

    public void saveGWASDataInferredView(OWLReasoner reasoner, File outputFile) throws OWLConversionException {
        try {
            // create new ontology to hold inferred axioms
            OWLOntology inferredOntology = getConverter().createConversionOntology();

            getLog().info("Saving inferred view...");
            List<InferredAxiomGenerator<? extends OWLAxiom>> gens =
                    new ArrayList<InferredAxiomGenerator<? extends OWLAxiom>>();
            // we require all inferred stuff except for disjoints...
            gens.add(new InferredClassAssertionAxiomGenerator());
            gens.add(new InferredDataPropertyCharacteristicAxiomGenerator());
            gens.add(new InferredEquivalentClassAxiomGenerator());
            gens.add(new InferredEquivalentDataPropertiesAxiomGenerator());
            gens.add(new InferredEquivalentObjectPropertyAxiomGenerator());
            gens.add(new InferredInverseObjectPropertiesAxiomGenerator());
            gens.add(new InferredObjectPropertyCharacteristicAxiomGenerator());
            gens.add(new InferredPropertyAssertionGenerator());
            gens.add(new InferredSubClassAxiomGenerator());
            gens.add(new InferredSubDataPropertyAxiomGenerator());
            gens.add(new InferredSubObjectPropertyAxiomGenerator());

            // now create the target ontology and save
            OWLOntologyManager inferredManager = inferredOntology.getOWLOntologyManager();
            InferredOntologyGenerator iog = new InferredOntologyGenerator(reasoner, gens);
            iog.fillOntology(inferredManager, inferredOntology);
            inferredManager.saveOntology(inferredOntology, IRI.create(outputFile));
            getLog().info("Inferred view saved ok");
        }
        catch (OWLOntologyStorageException e) {
            throw new OWLConversionException("Failed to save GWAS data (inferred view)", e);
        }
    }

    /**
     * Validates the data obtained from the GWAS catalog (prior to converting to OWL)
     *
     * @param studies the set of studies to validate
     */
    protected void validateGWASData(Collection<Study> studies) {
        // now check a random assortment of 5 studies for trait associations, abandoning broken ones
        int count = 0;
        int noAssocCount = 0;
        int termMismatches = 0;
        for (Study study : studies) {
//            try {
                Collection<Association> associations = study.getAssociations();
                getLog().debug("Study (PubMed ID '" + study.getPubmedId() + "') had " + associations.size() +
                                       " associations");
                if (associations.size() > 0) {
                    for (Association association : associations) {
                        String efoTraitsDashSepList="";
                        for(EfoTrait efoTrait : association.getEfoTraits()){
                            if("".equals(efoTraitsDashSepList)){
                                efoTraitsDashSepList.concat(efoTrait.getTrait());

                            }else {
                                efoTraitsDashSepList.concat(", " + efoTrait.getTrait());

                            }
                        }
                        for(Locus locus : association.getLoci()){
                            for(RiskAllele riskAllele : locus.getStrongestRiskAlleles()){
                                getLog().debug(
                                        //                                "    Association: SNP '" + association.getAssociatedSNP().getRSID() +
                                        "    Association: SNP '" + riskAllele.getSnp().getRsId() +
                                                "' <-> Trait '" +
                                                efoTraitsDashSepList.toString() + "'");
                            }
                        }
//                        getLog().debug(
//                                "    Association: SNP '" + association.getAssociatedSNP().getRSID() +
//                                        "' <-> Trait '" +
//                                        association.getAssociatedTrait().toString() + "'");
                    }
                    count++;
                }
                else {
                    noAssocCount++;
                }
//            }
//            catch (ObjectMappingException e) {
//                if (e instanceof OntologyTermException) {
//                    termMismatches++;
//                    getLog().error("EFO term mapping failed: " + e.getMessage());
//                }
//                else {
//                    getLog().warn("Excluding Study (PubMed ID '" + study.getPubmedId() + "'), " + e.getMessage());
//                }
//            }
        }
        int eligCount = studies.size() - noAssocCount;
        int correctCount = count + termMismatches;
        getLog().info("\n\nREPORT:\n" +
                              eligCount + "/" + studies.size() +
                              " declared associations and therefore could usefully be mapped.\n" +
                              (eligCount - count - termMismatches) + "/" + eligCount +
                              " failed due to data integrity concerns.\n" +
                              count + "/" + correctCount +
                              " studies could be completely mapped after passing all checks.\n" +
                              termMismatches + "/" + correctCount +
                              " failed due to missing or duplicated terms in EFO");
    }
}
{'fraction_non_alphanumeric': 0.09745527280808979, 'fraction_numerical': 0.0012501389043227025, 'mean_word_length': 2.1359003397508496, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2684255', 'n_tokens_mistral': 9536, 'n_tokens_neox': 8788, 'n_words': 1940}