<commit_before>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2009 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import org.apache.log4j.Logger;
import org.intermine.bio.ontology.OboParser;
import org.intermine.dataconversion.FileConverter;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.util.PropertiesUtil;
import org.intermine.util.StringUtil;
import org.intermine.xml.full.Item;
import org.intermine.xml.full.ReferenceList;

/**
 * DataConverter to parse a go annotation file into Items
 *
 *
 * @author <NAME>
 * @author <NAME> - some additions to record the parents of a go term.
 * @author <NAME> - changed evidence to be collection
 * @author <NAME> - refactored model
 */
public class GoConverter extends FileConverter
{
    protected static final String PROP_FILE = "go-annotation_config.properties";

    private OboParser oboParser = null;
    protected File ontology;

    protected Map<String, Item> goTerms = new LinkedHashMap<String, Item>();
    protected Map<GoTermToGene, Item> goAnnoItems;
    private Map<String, String> goEvidence = new LinkedHashMap<String, String>();
    private Map<String, Item> datasources = new LinkedHashMap<String, Item>();
    private Map<String, String> datasets = new LinkedHashMap<String, String>();
    private Map<String, String> publications = new LinkedHashMap<String, String>();
    private Map<String, Item> organisms = new LinkedHashMap<String, Item>();
    private Map<String, String> termIdNameMap = null;
    private Map<String, String> geneAttributes = new HashMap<String, String>();
    private Map<String, WithType> withTypes = new LinkedHashMap<String, WithType>();
    private Map<String, String> synonymTypes = new HashMap<String, String>();
    protected Map<String, ItemWrapper> productWrapperMap = new LinkedHashMap<String, ItemWrapper>();
    private Map<GoTermToGene, PlaceHolder> holderMap =
        new LinkedHashMap<GoTermToGene, PlaceHolder>();
    protected Set<String> productIds = new HashSet<String>();
    private static final Logger LOG = Logger.getLogger(GoConverter.class);
    protected IdResolverFactory resolverFactory;

    // TODO: datasources Map to contains ids not items? - need the dataset later on
    // TODO: store product after each one finished? - 'with' field may be a problem

    /**
     * Constructor
     *
     * @param writer the ItemWriter used to handle the resultant items
     * @param model the Model
     * @throws Exception if an error occurs in storing or finding Model
     */
    public GoConverter(ItemWriter writer, Model model) throws Exception {
        super(writer, model);
        addWithType("FB", "Gene", "primaryIdentifier");
        addWithType("UniProt", "Protein", "accession");
        synonymTypes.put("protein", "accession");
        synonymTypes.put("Protein", "accession");
        synonymTypes.put("gene", "identifier");
        synonymTypes.put("Gene", "identifier");

        // only construct factory here so can be replaced by mock factory in tests
        resolverFactory = new FlyBaseIdResolverFactory("gene");

        readConfig();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void close() throws Exception {
        //LOG.info("productWrapperMap.size() = " + productWrapperMap.size());
        // store all gene/protein data at the end of conversion because of  allGoAnnotation
        for (ItemWrapper nextWrapper : productWrapperMap.values()) {
            Item nextGeneProduct = nextWrapper.getItem();
            store(nextGeneProduct);
            store(nextWrapper.getSynonym());
        }
        super.close();
    }

    // read config file that has specific settings for each organism, key is taxon id
    private void readConfig() {
        Properties props = new Properties();
        try {
            props.load(getClass().getClassLoader().getResourceAsStream(PROP_FILE));
        } catch (IOException e) {
            throw new RuntimeException("Problem loading properties '" + PROP_FILE + "'", e);
        }
        Enumeration propNames = props.propertyNames();
        while (propNames.hasMoreElements()) {
            String taxonId = (String) propNames.nextElement();
            taxonId = taxonId.substring(0, taxonId.indexOf("."));
            Properties taxonProps = PropertiesUtil.stripStart(taxonId,
                PropertiesUtil.getPropertiesStartingWith(taxonId, props));
            String geneAttribute = taxonProps.getProperty("geneAttribute").trim();
            if (geneAttribute == null) {
                throw new IllegalArgumentException("Unable to find geneAttribute property for "
                                                   + "taxon: " + taxonId + " in file: "
                                                   + PROP_FILE);
            }
            if (!(geneAttribute.equals("identifier")
                            || geneAttribute.equals("primaryIdentifier"))) {
                throw new IllegalArgumentException("Invalid geneAttribute value for taxon: "
                                                   + taxonId + " was: " + geneAttribute);
            }
            geneAttributes.put(taxonId, geneAttribute);
        }
    }

    /**
     * Set the GO ontology file to be read from DAG format.
     *
     * @param ontology the GO ontology file
     */
    public void setOntologyfile(File ontology) {
        this.ontology = ontology;
    }

    /**
     * {@inheritDoc}
     */
    public void process(Reader reader) throws ObjectStoreException, IOException {

        // Renew this at the start of processing each file
        holderMap = new LinkedHashMap<GoTermToGene, PlaceHolder>();
        goAnnoItems = new LinkedHashMap<GoTermToGene, Item>();

        // on first call: parse ontology file and set up maps
        if (termIdNameMap == null) {
            if (ontology == null) {
                throw new IllegalArgumentException("ontology must be specified");
            }
            try {
                if (ontology.getName().endsWith(".obo")) {
                    oboParser = new OboParser();
                    termIdNameMap = oboParser.getTermIdNameMap(new FileReader(ontology));
                } else {
                    throw new IllegalArgumentException("Don't know how to deal with ontology file:"
                                                       + ontology.getAbsolutePath());
                }
            } catch (Exception e) {
                throw new IOException("" + e);
            }
        }

        BufferedReader br = new BufferedReader(reader);
        String line, lastProductId = null;

        // loop through entire file
        while ((line = br.readLine()) != null) {
            if (line.startsWith("!")) {
                continue;
            }
            String[] array = line.split("\t", -1); //keep trailing empty Strings
            if (array.length < 13) {
                throw new IllegalArgumentException("Not enough elements (should be > 13) in line: "
                                                   + line);
            }

            // We only want to create GOAnnotation objects applied to genes and proteins
            // some file entries apply to type 'transcript' and possibly others
            if (!("gene".equalsIgnoreCase(array[11])
                    || "protein".equalsIgnoreCase(array[11]))) {
                LOG.info("Ignored line with type: " + array[11]);
                continue;
            }

            String productId = array[1];

            // Wormbase has some proteins with UniProt accessions and some with WB:WP ids,<commit_msg>better error message <commit_after>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2009 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import org.apache.log4j.Logger;
import org.intermine.bio.ontology.OboParser;
import org.intermine.dataconversion.FileConverter;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.util.PropertiesUtil;
import org.intermine.util.StringUtil;
import org.intermine.xml.full.Item;
import org.intermine.xml.full.ReferenceList;

/**
 * DataConverter to parse a go annotation file into Items
 *
 *
 * @author <NAME>
 * @author <NAME> - some additions to record the parents of a go term.
 * @author <NAME> - changed evidence to be collection
 * @author <NAME> - refactored model
 */
public class GoConverter extends FileConverter
{
    protected static final String PROP_FILE = "go-annotation_config.properties";

    private OboParser oboParser = null;
    protected File ontology;

    protected Map<String, Item> goTerms = new LinkedHashMap<String, Item>();
    protected Map<GoTermToGene, Item> goAnnoItems;
    private Map<String, String> goEvidence = new LinkedHashMap<String, String>();
    private Map<String, Item> datasources = new LinkedHashMap<String, Item>();
    private Map<String, String> datasets = new LinkedHashMap<String, String>();
    private Map<String, String> publications = new LinkedHashMap<String, String>();
    private Map<String, Item> organisms = new LinkedHashMap<String, Item>();
    private Map<String, String> termIdNameMap = null;
    private Map<String, String> geneAttributes = new HashMap<String, String>();
    private Map<String, WithType> withTypes = new LinkedHashMap<String, WithType>();
    private Map<String, String> synonymTypes = new HashMap<String, String>();
    protected Map<String, ItemWrapper> productWrapperMap = new LinkedHashMap<String, ItemWrapper>();
    private Map<GoTermToGene, PlaceHolder> holderMap =
        new LinkedHashMap<GoTermToGene, PlaceHolder>();
    protected Set<String> productIds = new HashSet<String>();
    private static final Logger LOG = Logger.getLogger(GoConverter.class);
    protected IdResolverFactory resolverFactory;

    // TODO: datasources Map to contains ids not items? - need the dataset later on
    // TODO: store product after each one finished? - 'with' field may be a problem

    /**
     * Constructor
     *
     * @param writer the ItemWriter used to handle the resultant items
     * @param model the Model
     * @throws Exception if an error occurs in storing or finding Model
     */
    public GoConverter(ItemWriter writer, Model model) throws Exception {
        super(writer, model);
        addWithType("FB", "Gene", "primaryIdentifier");
        addWithType("UniProt", "Protein", "accession");
        synonymTypes.put("protein", "accession");
        synonymTypes.put("Protein", "accession");
        synonymTypes.put("gene", "identifier");
        synonymTypes.put("Gene", "identifier");

        // only construct factory here so can be replaced by mock factory in tests
        resolverFactory = new FlyBaseIdResolverFactory("gene");

        readConfig();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void close() throws Exception {
        //LOG.info("productWrapperMap.size() = " + productWrapperMap.size());
        // store all gene/protein data at the end of conversion because of  allGoAnnotation
        for (ItemWrapper nextWrapper : productWrapperMap.values()) {
            Item nextGeneProduct = nextWrapper.getItem();
            store(nextGeneProduct);
            store(nextWrapper.getSynonym());
        }
        super.close();
    }

    // read config file that has specific settings for each organism, key is taxon id
    private void readConfig() {
        Properties props = new Properties();
        try {
            props.load(getClass().getClassLoader().getResourceAsStream(PROP_FILE));
        } catch (IOException e) {
            throw new RuntimeException("Problem loading properties '" + PROP_FILE + "'", e);
        }
        Enumeration propNames = props.propertyNames();
        while (propNames.hasMoreElements()) {
            String taxonId = (String) propNames.nextElement();
            taxonId = taxonId.substring(0, taxonId.indexOf("."));
            Properties taxonProps = PropertiesUtil.stripStart(taxonId,
                PropertiesUtil.getPropertiesStartingWith(taxonId, props));
            String geneAttribute = taxonProps.getProperty("geneAttribute").trim();
            if (geneAttribute == null) {
                throw new IllegalArgumentException("Unable to find geneAttribute property for "
                                                   + "taxon: " + taxonId + " in file: "
                                                   + PROP_FILE);
            }
            if (!(geneAttribute.equals("identifier")
                            || geneAttribute.equals("primaryIdentifier"))) {
                throw new IllegalArgumentException("Invalid geneAttribute value for taxon: "
                                                   + taxonId + " was: " + geneAttribute);
            }
            geneAttributes.put(taxonId, geneAttribute);
        }
    }

    /**
     * Set the GO ontology file to be read from DAG format.
     *
     * @param ontology the GO ontology file
     */
    public void setOntologyfile(File ontology) {
        this.ontology = ontology;
    }

    /**
     * {@inheritDoc}
     */
    public void process(Reader reader) throws ObjectStoreException, IOException {

        // Renew this at the start of processing each file
        holderMap = new LinkedHashMap<GoTermToGene, PlaceHolder>();
        goAnnoItems = new LinkedHashMap<GoTermToGene, Item>();

        // on first call: parse ontology file and set up maps
        if (termIdNameMap == null) {
            if (ontology == null) {
                throw new IllegalArgumentException("ontology must be specified");
            }
            try {
                if (ontology.getName().endsWith(".obo")) {
                    oboParser = new OboParser();
                    termIdNameMap = oboParser.getTermIdNameMap(new FileReader(ontology));
                } else {
                    throw new IllegalArgumentException("Don't know how to deal with ontology file:"
                                                       + ontology.getAbsolutePath());
                }
            } catch (Exception e) {
                throw new IOException("" + e);
            }
        }

        BufferedReader br = new BufferedReader(reader);
        String line, lastProductId = null;

        // loop through entire file
        while ((line = br.readLine()) != null) {
            if (line.startsWith("!")) {
                continue;
            }
            String[] array = line.split("\t", -1); //keep trailing empty Strings
            if (array.length < 13) {
                throw new IllegalArgumentException("Not enough elements (should be > 13 not " +
                                                   array.length + ") in line: " + line);
            }

            // We only want to create GOAnnotation objects applied to genes and proteins
            // some file entries apply to type 'transcript' and possibly others
            if (!("gene".equalsIgnoreCase(array[11])
                    || "protein".equalsIgnoreCase(array[11]))) {
                LOG.info("Ignored line with type: " + array[11]);
                continue;
            }

            String productId = array[1];

            // Wormbase has some proteins with UniProt accessions and some with WB:WP ids,
{'fraction_non_alphanumeric': 0.09951412632715494, 'fraction_numerical': 0.0026393137784176115, 'mean_word_length': 2.451045332229352, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2657623', 'n_tokens_mistral': 4291, 'n_tokens_neox': 4117, 'n_words': 1272}