<commit_before>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2009 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.File;
import java.io.FileReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.DirectoryConverter;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.util.SAXParser;
import org.intermine.util.StringUtil;
import org.intermine.util.Util;
import org.intermine.xml.full.Item;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;


/**
 * DataConverter to parse UniProt data into items.  Improved version of UniProtConverter.
 *
 * Differs from UniProtConverter in that this Converter creates proper protein items.
 * UniProtConverter creates protein objects, that are really uniprot entries.
 * @author <NAME>
 */
public class UniprotConverter extends DirectoryConverter
{
    private static final UniprotConfig CONFIG = new UniprotConfig();
    private static final Logger LOG = Logger.getLogger(UniprotConverter.class);
    private Map<String, String> pubs = new HashMap();
    private Map<String, String> organisms = new HashMap();
    private Map<String, String> comments = new HashMap();
    private Map<String, String> synonyms = new HashMap();
    private Map<String, String> datasets = new HashMap();
    private Map<String, String> domains = new HashMap();
    private Map<String, List<String>> sequences = new HashMap();
    private Map<String, String> datasources = new HashMap();
    private Map<String, String> ontologies = new HashMap();
    private Map<String, String> keywords = new HashMap();
    private Map<String, String> genes = new HashMap();
    // don't allow duplicate identifiers
    private List<String> geneIdentifiers = new ArrayList();

    private boolean createInterpro = false;
    private Set<String> taxonIds = null;

    protected IdResolverFactory resolverFactory;
    private IdResolver resolver;
    private String datasourceRefId = null;

    /**
     * Constructor
     * @param writer the ItemWriter used to handle the resultant items
     * @param model the Model
     */
    public UniprotConverter(ItemWriter writer, Model model) {
        super(writer, model);
        // only construct factory here so can be replaced by mock factory in tests
        resolverFactory = new FlyBaseIdResolverFactory("gene");
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void process(File dataDir) throws Exception {
        Map<String, File[]> taxonIdToFiles = parseFileNames(dataDir.listFiles());
        // check that we have valid files before we start storing ANY data
        if (taxonIdToFiles == null || taxonIdToFiles.isEmpty()) {
            throw new RuntimeException("no files found in " + dataDir.getCanonicalPath());
        }
        try {
            datasourceRefId = getDataSource("UniProt");
            setOntology("UniProtKeyword");
        } catch (SAXException e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        }
        Iterator iter = taxonIds.iterator();
        while (iter.hasNext()) {
            String taxonId = iter.next().toString();
            if (taxonIdToFiles.get(taxonId) == null) {
                throw new RuntimeException("no files found for " + taxonId);
            }
            File[] files = taxonIdToFiles.get(taxonId);
            for (int i = 0; i <= 1; i++) {
                File file = files[i];
                if (file == null) {
                    continue;
                }
                Set<UniprotEntry> entries = new HashSet();
                UniprotHandler handler = new UniprotHandler(entries);
                try {
                    Reader reader = new FileReader(file);
                    SAXParser.parse(new InputSource(reader), handler);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
                // process all uniprot entries
                Set<UniprotEntry> isoforms = processEntries(entries);
                // process all isoforms created
                processEntries(isoforms);
            }

            // reset all variables here, new organism
            synonyms = new HashMap();
            sequences = new HashMap();
            genes = new HashMap();
        }
    }

    /*
     * sprot data files need to be processed immediately before trembl ones
     * not all organisms are going to have both files
     *
     * UniProtFilterTask has already been run so all files are assumed to be valid.
     *
     *  expected syntax : 7227_uniprot_sprot.xml
     *  [TAXONID]_uniprot_[SOURCE].xml
     *  SOURCE: sprot or trembl
     */
    private Map<String, File[]> parseFileNames(File[] fileList) {
        Map<String, File[]> files = new HashMap();
        if (fileList == null) {
            return null;
        }
        for (File file : fileList) {
            String[] bits = file.getName().split("_");
            String taxonId = bits[0];
            if (bits.length != 3) {
                LOG.info("Bad file found:  "  + file.getName()
                         + ", expected a filename like 7227_uniprot_sprot.xml");
                continue;
            }
            String source = bits[2].replace(".xml", "");
            // process trembl last because trembl has duplicates of sprot proteins
            if (!source.equals("sprot") && !source.equals("trembl")) {
                LOG.info("Bad file found:  "  + file.getName()
                         +  " (" + bits[2] + "), expecting sprot or trembl ");
                continue;
            }
            int i = (source.equals("sprot") ? 0 : 1);
            if (!files.containsKey(taxonId)) {
                File[] sourceFiles = new File[2];
                sourceFiles[i] = file;
                files.put(taxonId, sourceFiles);
            } else {
                files.get(taxonId)[i] = file;
            }
        }
        return files;
    }

    /**
     * Toggle whether or not to import interpro data
     * @param createinterpro String whether or not to import interpro data (true/false)
     */
    public void setCreateinterpro(String createinterpro) {
        if (createinterpro.equals("true")) {
            this.createInterpro = true;
        } else {
            this.createInterpro = false;
        }
    }

    /**
     * Sets the list of taxonIds that should be imported if using split input files.
     *
     * @param taxonIds a space-separated list of taxonIds
     */
    public void setUniprotOrganisms(String taxonIds) {
        this.taxonIds = new HashSet<String>(Arrays.asList(StringUtil.split(taxonIds, " ")));
        LOG.info("Setting list of organisms to " + this.taxonIds);
    }

    private Set<UniprotEntry> processEntries(Set<UniprotEntry> entries)
    throws SAXException {

        Set<UniprotEntry> isoforms = new HashSet();

        for (UniprotEntry entry : entries) {
            // TODO there are uniparc entries so check for swissprot-trembl datasets
            if (entry.hasDatasetRefId() && entry.hasPrimaryAccession() && !entry.isDuplicate()) {

                for (String isoformAccession: entry.getIsoforms()) {
                    isoforms.add(entry.clone(isoformAccession));
                }

                Item protein = createItem("Protein");

                /* primaryAccession, primaryIdentifier, name, etc */
                processIdentifiers(protein, entry);

                String isCanonical = (entry.isIsoform() ? "false" : "true");
                protein.setAttribute("isUniprotCanonical", isCanonical);

                /* dataset */
                protein.addToCollection("dataSets", entry.getDatasetRefId());

                /* sequence */
                if (!entry.isIsoform()) {
                    processSequence(protein, entry);
                }

                /* interpro */
                if (createInterpro && !entry.getDomains().isEmpty()) {
                    protein.setCollection("proteinDomains", entry.getDomains());
                }

                /* organism */
                try {
                    protein.setReference("organism", getOrganism(entry.getTaxonId()));
                } catch (SAXException e) {
                    throw new RuntimeException("store failed for " + entry.getPrimaryAccession());
                }

                /* publications */
                if (!entry.getPubs().isEmpty()) {
                    protein.setCollection("publications", entry.getPubs());
                }

                /* comments */
                if (!entry.getComments().isEmpty()) {
                    protein.setCollection("comments", entry.getComments());
                }

                /* keywords */
                if (!entry.getKeywords().isEmpty()) {
                    protein.setCollection("keywords", entry.getKeywords());
                }

                /* features */
                if (!entry.getFeatures().isEmpty() && !entry.isIsoform()) {
                    protein.setCollection("features", entry.getFeatures());
                }

                /* components */
                if (!entry.getComponents().isEmpty()) {
                    processComponents(protein, entry);
                }

                List<String> synonymRefIds = new ArrayList();
                try {
                    processDbrefs(protein, entry, synonymRefIds);

                    /* genes */
                    processGene(protein, entry);

                    store(protein);

                    processSynonyms(synonymRefIds, protein.getIdentifier(), entry);
                    if (!synonymRefIds.isEmpty()) {
                        protein.setCollection("synonyms", synonymRefIds);
                    }

                } catch (ObjectStoreException e) {
                    throw new SAXException(e);
                }

            }
        }
        return isoforms;
    }

    private void processSequence(Item protein, UniprotEntry entry) {
        protein.setAttribute("length", entry.getLength());
        protein.setReference("sequence", entry.getSeqRefId());
        protein.setAttribute("molecularWeight", entry.getMolecularWeight());
        protein.setAttribute("md5checksum", entry.getMd5checksum());
    }

    private void processIdentifiers(Item protein, UniprotEntry entry) {
        protein.setAttribute("name", entry.getName());
        protein.setAttribute("isFragment", entry.isFragment());
        protein.setAttribute("uniprotAccession", entry.getUniprotAccession());
        String primaryAccession = entry.getPrimaryAccession();
        protein.setAttribute("primaryAccession", primaryAccession);

        String primaryIdentifier = entry.getPrimaryIdentifier();
        protein.setAttribute("uniprotName", primaryIdentifier);

        // primaryIdentifier must be unique, so append isoform suffix, eg -1
        if (entry.isIsoform()) {
            primaryIdentifier = getIsoformIdentifier(primaryAccession, primaryIdentifier);
        }
        protein.setAttribute("primaryIdentifier", primaryIdentifier);
    }

    private String getIsoformIdentifier(String primaryAccession, String primaryIdentifier) {
        String isoformIdentifier = primaryIdentifier;
        String[] bits = primaryAccession.split("\\-");
        if (bits.length == 2) {
            isoformIdentifier += "-" + bits[1];
        }
        return isoformIdentifier;
    }

    private void processComponents(Item protein, UniprotEntry entry)
    throws SAXException {
        for (String componentName : entry.getComponents()) {
            Item component = createItem("Component");
            component.setAttribute("name", componentName);
            component.setReference("protein", protein);
            try {
                store(component);
            } catch (ObjectStoreException e) {
                throw new SAXException(e);
            }
        }
    }

    private void processSynonyms(List<String> proteinSynonyms, String proteinRefId,
                                 UniprotEntry entry)
    throws SAXException {

        String datasetRefId = entry.getDatasetRefId();

        // primary accession
        String refId = getSynonym(proteinRefId, "accession", entry.getPrimaryAccession(), "true",
                                  datasetRefId);
        proteinSynonyms.add(refId);

        // accessions
        for (String accession : entry.getAccessions()) {
            refId = getSynonym(proteinRefId, "accession", accession, "false", datasetRefId);
            proteinSynonyms.add(refId);
        }

        // primaryIdentifier
        String primaryIdentifier = entry.getPrimaryIdentifier();
        refId = getSynonym(proteinRefId, "identifier", primaryIdentifier, "false", datasetRefId);
        proteinSynonyms.add(refId);

        // primaryIdentifier if isoform
        if (entry.isIsoform()) {
                String isoformIdentifier
                = getIsoformIdentifier(entry.getPrimaryAccession(), entry.getPrimaryIdentifier());
                refId = getSynonym(proteinRefId, "identifier", isoformIdentifier, "false",
                                   datasetRefId);
                proteinSynonyms.add(refId);
        }

        // name <recommendedName> or <alternateName>
        for (String name : entry.getProteinNames()) {
            refId = getSynonym(proteinRefId, "name", name, "false", datasetRefId);
            proteinSynonyms.add(refId);
        }

        // duplicate trembl entries
        if (!entry.isIsoform() && entry.getMd5checksum() != null
                        && !sequences.get(entry.getMd5checksum()).isEmpty()) {
            for (String synonym : sequences.get(entry.getMd5checksum())) {
                refId = getSynonym(proteinRefId, "accession", synonym, "false", datasetRefId);
                proteinSynonyms.add(refId);
            }
        }

        // isoforms with extra identifiers
        List<String> isoformSynonyms = entry.getIsoformSynonyms();
        if (!isoformSynonyms.isEmpty()) {
            for (String synonym : isoformSynonyms) {
                refId = getSynonym(proteinRefId, "accession", synonym, "false", datasetRefId);
                proteinSynonyms.add(refId);
            }
        }
    }

    private void processDbrefs(Item protein, UniprotEntry entry, List<String> synonymRefIds)
    throws SAXException {
        Map<String, String> dbrefs = entry.getDbrefs();
        if (dbrefs.containsKey("EC")) {
            protein.setAttribute("ecNumber", dbrefs.get("EC"));
        }
        if (dbrefs.containsKey("RefSeq")) {
            String synonym = dbrefs.get("RefSeq");
            String refId = getSynonym(protein.getIdentifier(), "identifier", synonym, "false",
                                      entry.getDatasetRefId());
            synonymRefIds.add(refId);
        }
    }

    // gets the unique identifier and list of identifiers to set
    // loops through each gene entry, assigns refId to protein
    private void processGene(Item protein, UniprotEntry entry)
    throws SAXException {
        String taxonId = entry.getTaxonId();
        // which gene.identifier field has to be unique
        String uniqueIdentifierField = CONFIG.getUniqueIdentifier(taxonId);
        if (uniqueIdentifierField == null) {
            LOG.error("Couldn't process genes for " + taxonId + ", no unique identifier set");
            return;
        }

        // for this organism, set the following gene fields
        Set<String> geneFields = CONFIG.getGeneIdentifierFields(taxonId);

        if (geneFields == null) {
            LOG.error("Couldn't process genes for " + taxonId + ", configuration missing");
            return;
        }

        // just one gene, don't have to worry about gene designations and dbrefs
        if (!entry.hasMultipleGenes()) {
            String geneRefId = createGene(entry, taxonId, geneFields, uniqueIdentifierField);
            if (geneRefId != null) {
                protein.addToCollection("genes", geneRefId);
            }
            return;
        }

        // loop through each gene entry to be processed
        // cloning the gene removes dbrefs without gene designations
        List<UniprotEntry> clonedEntries = entry.cloneGenes();
        Iterator<UniprotEntry> iter = clonedEntries.iterator();
        while (iter.hasNext()) {
            // create a dummy entry and add identifiers for specific gene
            String geneRefId = createGene(iter.next(), taxonId, geneFields, uniqueIdentifierField);
            if (!StringUtil.isEmpty(geneRefId)) {
                protein.addToCollection("genes", geneRefId);
            }
        }
    }

    // creates and stores the gene
    // sets the identifier fields specified in the config file
    // creates synonym
    private String createGene(UniprotEntry entry, String taxonId, Set<String> geneFields,
                              String uniqueIdentifierFieldType)
    throws SAXException {
        List<String> geneSynonyms = new ArrayList();

        String uniqueIdentifierValue = getGeneIdentifier(entry, taxonId, uniqueIdentifierFieldType,
                                                    geneSynonyms, true);
        if (uniqueIdentifierValue == null) {
            return null;
        }
        String geneRefId = genes.get(uniqueIdentifierValue);
        if (geneRefId == null) {
            Item gene = createItem("Gene");
            genes.put(uniqueIdentifierValue, gene.getIdentifier());
            gene.addToCollection("dataSets", entry.getDatasetRefId());
            gene.setAttribute(uniqueIdentifierFieldType, uniqueIdentifierValue);

            // set each identifier
            for (String geneField : geneFields) {
                if (geneField.equals(uniqueIdentifierFieldType)) {
                    // we've already set the key field
                    continue;
                }
                String identifier = getGeneIdentifier(entry, taxonId, geneField, geneSynonyms,
                                                      false);

                if (identifier == null) {
                    LOG.error("Couldn't process gene, no " + geneField);
                    continue;
                }

                /*
                 * if the protein is an isoform, this gene has already been processed so the
                 * identifier will always be a duplicate in this case.
                 */
                if (!entry.isIsoform() && geneIdentifiers.contains(identifier)) {
                    LOG.error("not assigning duplicate identifier:  " + identifier);
                    continue;
                    // if the canonical protein is processed and the gene has a duplicate
                    // identifier, we need to flag so the gene won't be created for the isoform
                    // either.
                }
                geneIdentifiers.add(identifier);
                gene.setAttribute(geneField, identifier);
            }

            // store gene
            try {
                gene.setReference("organism", getOrganism(taxonId));
                store(gene);
            } catch (ObjectStoreException e) {
                throw new SAXException(e);
            }

            // synonyms
            geneRefId = gene.getIdentifier();
            for (String identifier : geneSynonyms) {
                getSynonym(geneRefId, "identifier", identifier, "false", entry.getDatasetRefId());
            }
            getSynonym(geneRefId, "identifier", uniqueIdentifierValue, "false",
                       entry.getDatasetRefId());
        }
        return geneRefId;
    }

    // gets the identifier for a gene from the dbref/names collected from the XML
    // which identifier is chosen depends on the configuration in the uniprot config file
    private String getGeneIdentifier(UniprotEntry entry, String taxonId, String identifierType,
                                     List<String> geneSynonyms, boolean isUniqueIdentifier) {

        String identifierValue = null;
        // how to get the identifier, eg. dbref OR name
        String method = CONFIG.getIdentifierMethod(taxonId, identifierType);
        // what value to use with method, eg. "FlyBase" or "ORF"
        String value = CONFIG.getIdentifierValue(taxonId, identifierType);

        if (method == null || value == null) {
            // use default set in config file, if this organism isn't configured
            method = CONFIG.getIdentifierMethod("default", identifierType);
            value = CONFIG.getIdentifierValue("default", identifierType);
            if (method == null || value == null) {
                throw new RuntimeException("error processing line in config file for organism "
                                           + taxonId);
            }
        }

        if (method.equals("name")) {
            if (entry.getGeneNames() == null || entry.getGeneNames().isEmpty()) {
                LOG.error(" ~~~ No gene names for " + taxonId + ". protein accession:"
                          + entry.getPrimaryAccession());
                return null;
            }
            identifierValue = entry.getGeneNames().get(value);
        } else if (method.equals("dbref")) {
            identifierValue = entry.getDbrefs().get(value);
        } else {
            LOG.error("error processing line in config file for organism " + taxonId);
            return null;
        }
        geneSynonyms.add(identifierValue);
        if (isUniqueIdentifier) {
            identifierValue = resolveGene(taxonId, identifierValue);

            // try again!
            if (identifierValue == null && entry.getGeneNames() != null
                            && !entry.getGeneNames().isEmpty()) {
                Iterator<String> iter = entry.getGeneNames().values().iterator();
                while (iter.hasNext() && identifierValue == null) {
                    identifierValue = resolveGene(taxonId, iter.next());
                }
            }
        }
        return identifierValue;
    }

    private String resolveGene(String taxonId, String identifier) {
        resolver = resolverFactory.getIdResolver(false);
        if (resolver == null) {
            // no id resolver available, so return the original identifier
            return identifier;
        }
        int resCount = resolver.countResolutions(taxonId, identifier);
        if (resCount != 1) {
            LOG.info("RESOLVER: failed to resolve gene to one identifier, ignoring gene: "
                     + identifier + " count: " + resCount + " FBgn: "
                     + resolver.resolveId(taxonId, identifier));
            return null;
        }
        return resolver.resolveId(taxonId, identifier).iterator().next();
    }

    /* converts the XML into UniProt entry objects.  run once per file */
    private class UniprotHandler extends DefaultHandler
    {
        private UniprotEntry entry;
        private Stack<String> stack = new Stack<String>();
        private String attName = null;
        private StringBuffer attValue = null;
        private String taxonId = null;
        private Set<UniprotEntry> entries;

        /**
         * @param entries empty map to be populated with uniprot entries
         * @param isoforms empty map to be populated with isoforms
         */
        public UniprotHandler(Set<UniprotEntry> entries) {
            this.entries = entries;<commit_msg>resolver is only used for fly for now.<commit_after>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2009 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.File;
import java.io.FileReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.DirectoryConverter;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.util.SAXParser;
import org.intermine.util.StringUtil;
import org.intermine.util.Util;
import org.intermine.xml.full.Item;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;


/**
 * DataConverter to parse UniProt data into items.  Improved version of UniProtConverter.
 *
 * Differs from UniProtConverter in that this Converter creates proper protein items.
 * UniProtConverter creates protein objects, that are really uniprot entries.
 * @author <NAME>
 */
public class UniprotConverter extends DirectoryConverter
{
    private static final UniprotConfig CONFIG = new UniprotConfig();
    private static final Logger LOG = Logger.getLogger(UniprotConverter.class);
    private Map<String, String> pubs = new HashMap();
    private Map<String, String> organisms = new HashMap();
    private Map<String, String> comments = new HashMap();
    private Map<String, String> synonyms = new HashMap();
    private Map<String, String> datasets = new HashMap();
    private Map<String, String> domains = new HashMap();
    private Map<String, List<String>> sequences = new HashMap();
    private Map<String, String> datasources = new HashMap();
    private Map<String, String> ontologies = new HashMap();
    private Map<String, String> keywords = new HashMap();
    private Map<String, String> genes = new HashMap();
    // don't allow duplicate identifiers
    private List<String> geneIdentifiers = new ArrayList();

    private boolean createInterpro = false;
    private Set<String> taxonIds = null;

    protected IdResolverFactory resolverFactory;
    private IdResolver flyResolver;
    private String datasourceRefId = null;

    /**
     * Constructor
     * @param writer the ItemWriter used to handle the resultant items
     * @param model the Model
     */
    public UniprotConverter(ItemWriter writer, Model model) {
        super(writer, model);
        // only construct factory here so can be replaced by mock factory in tests
        resolverFactory = new FlyBaseIdResolverFactory("gene");
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void process(File dataDir) throws Exception {
        Map<String, File[]> taxonIdToFiles = parseFileNames(dataDir.listFiles());
        // check that we have valid files before we start storing ANY data
        if (taxonIdToFiles == null || taxonIdToFiles.isEmpty()) {
            throw new RuntimeException("no files found in " + dataDir.getCanonicalPath());
        }
        try {
            datasourceRefId = getDataSource("UniProt");
            setOntology("UniProtKeyword");
        } catch (SAXException e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        }
        Iterator iter = taxonIds.iterator();
        while (iter.hasNext()) {
            String taxonId = iter.next().toString();
            if (taxonIdToFiles.get(taxonId) == null) {
                throw new RuntimeException("no files found for " + taxonId);
            }
            File[] files = taxonIdToFiles.get(taxonId);
            for (int i = 0; i <= 1; i++) {
                File file = files[i];
                if (file == null) {
                    continue;
                }
                Set<UniprotEntry> entries = new HashSet();
                UniprotHandler handler = new UniprotHandler(entries);
                try {
                    Reader reader = new FileReader(file);
                    SAXParser.parse(new InputSource(reader), handler);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
                // process all uniprot entries
                Set<UniprotEntry> isoforms = processEntries(entries);
                // process all isoforms created
                processEntries(isoforms);
            }

            // reset all variables here, new organism
            synonyms = new HashMap();
            sequences = new HashMap();
            genes = new HashMap();
        }
    }

    /*
     * sprot data files need to be processed immediately before trembl ones
     * not all organisms are going to have both files
     *
     * UniProtFilterTask has already been run so all files are assumed to be valid.
     *
     *  expected syntax : 7227_uniprot_sprot.xml
     *  [TAXONID]_uniprot_[SOURCE].xml
     *  SOURCE: sprot or trembl
     */
    private Map<String, File[]> parseFileNames(File[] fileList) {
        Map<String, File[]> files = new HashMap();
        if (fileList == null) {
            return null;
        }
        for (File file : fileList) {
            String[] bits = file.getName().split("_");
            String taxonId = bits[0];
            if (bits.length != 3) {
                LOG.info("Bad file found:  "  + file.getName()
                         + ", expected a filename like 7227_uniprot_sprot.xml");
                continue;
            }
            String source = bits[2].replace(".xml", "");
            // process trembl last because trembl has duplicates of sprot proteins
            if (!source.equals("sprot") && !source.equals("trembl")) {
                LOG.info("Bad file found:  "  + file.getName()
                         +  " (" + bits[2] + "), expecting sprot or trembl ");
                continue;
            }
            int i = (source.equals("sprot") ? 0 : 1);
            if (!files.containsKey(taxonId)) {
                File[] sourceFiles = new File[2];
                sourceFiles[i] = file;
                files.put(taxonId, sourceFiles);
            } else {
                files.get(taxonId)[i] = file;
            }
        }
        return files;
    }

    /**
     * Toggle whether or not to import interpro data
     * @param createinterpro String whether or not to import interpro data (true/false)
     */
    public void setCreateinterpro(String createinterpro) {
        if (createinterpro.equals("true")) {
            this.createInterpro = true;
        } else {
            this.createInterpro = false;
        }
    }

    /**
     * Sets the list of taxonIds that should be imported if using split input files.
     *
     * @param taxonIds a space-separated list of taxonIds
     */
    public void setUniprotOrganisms(String taxonIds) {
        this.taxonIds = new HashSet<String>(Arrays.asList(StringUtil.split(taxonIds, " ")));
        LOG.info("Setting list of organisms to " + this.taxonIds);
    }

    private Set<UniprotEntry> processEntries(Set<UniprotEntry> entries)
    throws SAXException {

        Set<UniprotEntry> isoforms = new HashSet();

        for (UniprotEntry entry : entries) {
            // TODO there are uniparc entries so check for swissprot-trembl datasets
            if (entry.hasDatasetRefId() && entry.hasPrimaryAccession() && !entry.isDuplicate()) {

                for (String isoformAccession: entry.getIsoforms()) {
                    isoforms.add(entry.clone(isoformAccession));
                }

                Item protein = createItem("Protein");

                /* primaryAccession, primaryIdentifier, name, etc */
                processIdentifiers(protein, entry);

                String isCanonical = (entry.isIsoform() ? "false" : "true");
                protein.setAttribute("isUniprotCanonical", isCanonical);

                /* dataset */
                protein.addToCollection("dataSets", entry.getDatasetRefId());

                /* sequence */
                if (!entry.isIsoform()) {
                    processSequence(protein, entry);
                }

                /* interpro */
                if (createInterpro && !entry.getDomains().isEmpty()) {
                    protein.setCollection("proteinDomains", entry.getDomains());
                }

                /* organism */
                try {
                    protein.setReference("organism", getOrganism(entry.getTaxonId()));
                } catch (SAXException e) {
                    throw new RuntimeException("store failed for " + entry.getPrimaryAccession());
                }

                /* publications */
                if (!entry.getPubs().isEmpty()) {
                    protein.setCollection("publications", entry.getPubs());
                }

                /* comments */
                if (!entry.getComments().isEmpty()) {
                    protein.setCollection("comments", entry.getComments());
                }

                /* keywords */
                if (!entry.getKeywords().isEmpty()) {
                    protein.setCollection("keywords", entry.getKeywords());
                }

                /* features */
                if (!entry.getFeatures().isEmpty() && !entry.isIsoform()) {
                    protein.setCollection("features", entry.getFeatures());
                }

                /* components */
                if (!entry.getComponents().isEmpty()) {
                    processComponents(protein, entry);
                }

                List<String> synonymRefIds = new ArrayList();
                try {
                    processDbrefs(protein, entry, synonymRefIds);

                    /* genes */
                    processGene(protein, entry);

                    store(protein);

                    processSynonyms(synonymRefIds, protein.getIdentifier(), entry);
                    if (!synonymRefIds.isEmpty()) {
                        protein.setCollection("synonyms", synonymRefIds);
                    }

                } catch (ObjectStoreException e) {
                    throw new SAXException(e);
                }

            }
        }
        return isoforms;
    }

    private void processSequence(Item protein, UniprotEntry entry) {
        protein.setAttribute("length", entry.getLength());
        protein.setReference("sequence", entry.getSeqRefId());
        protein.setAttribute("molecularWeight", entry.getMolecularWeight());
        protein.setAttribute("md5checksum", entry.getMd5checksum());
    }

    private void processIdentifiers(Item protein, UniprotEntry entry) {
        protein.setAttribute("name", entry.getName());
        protein.setAttribute("isFragment", entry.isFragment());
        protein.setAttribute("uniprotAccession", entry.getUniprotAccession());
        String primaryAccession = entry.getPrimaryAccession();
        protein.setAttribute("primaryAccession", primaryAccession);

        String primaryIdentifier = entry.getPrimaryIdentifier();
        protein.setAttribute("uniprotName", primaryIdentifier);

        // primaryIdentifier must be unique, so append isoform suffix, eg -1
        if (entry.isIsoform()) {
            primaryIdentifier = getIsoformIdentifier(primaryAccession, primaryIdentifier);
        }
        protein.setAttribute("primaryIdentifier", primaryIdentifier);
    }

    private String getIsoformIdentifier(String primaryAccession, String primaryIdentifier) {
        String isoformIdentifier = primaryIdentifier;
        String[] bits = primaryAccession.split("\\-");
        if (bits.length == 2) {
            isoformIdentifier += "-" + bits[1];
        }
        return isoformIdentifier;
    }

    private void processComponents(Item protein, UniprotEntry entry)
    throws SAXException {
        for (String componentName : entry.getComponents()) {
            Item component = createItem("Component");
            component.setAttribute("name", componentName);
            component.setReference("protein", protein);
            try {
                store(component);
            } catch (ObjectStoreException e) {
                throw new SAXException(e);
            }
        }
    }

    private void processSynonyms(List<String> proteinSynonyms, String proteinRefId,
                                 UniprotEntry entry)
    throws SAXException {

        String datasetRefId = entry.getDatasetRefId();

        // primary accession
        String refId = getSynonym(proteinRefId, "accession", entry.getPrimaryAccession(), "true",
                                  datasetRefId);
        proteinSynonyms.add(refId);

        // accessions
        for (String accession : entry.getAccessions()) {
            refId = getSynonym(proteinRefId, "accession", accession, "false", datasetRefId);
            proteinSynonyms.add(refId);
        }

        // primaryIdentifier
        String primaryIdentifier = entry.getPrimaryIdentifier();
        refId = getSynonym(proteinRefId, "identifier", primaryIdentifier, "false", datasetRefId);
        proteinSynonyms.add(refId);

        // primaryIdentifier if isoform
        if (entry.isIsoform()) {
                String isoformIdentifier
                = getIsoformIdentifier(entry.getPrimaryAccession(), entry.getPrimaryIdentifier());
                refId = getSynonym(proteinRefId, "identifier", isoformIdentifier, "false",
                                   datasetRefId);
                proteinSynonyms.add(refId);
        }

        // name <recommendedName> or <alternateName>
        for (String name : entry.getProteinNames()) {
            refId = getSynonym(proteinRefId, "name", name, "false", datasetRefId);
            proteinSynonyms.add(refId);
        }

        // duplicate trembl entries
        if (!entry.isIsoform() && entry.getMd5checksum() != null
                        && !sequences.get(entry.getMd5checksum()).isEmpty()) {
            for (String synonym : sequences.get(entry.getMd5checksum())) {
                refId = getSynonym(proteinRefId, "accession", synonym, "false", datasetRefId);
                proteinSynonyms.add(refId);
            }
        }

        // isoforms with extra identifiers
        List<String> isoformSynonyms = entry.getIsoformSynonyms();
        if (!isoformSynonyms.isEmpty()) {
            for (String synonym : isoformSynonyms) {
                refId = getSynonym(proteinRefId, "accession", synonym, "false", datasetRefId);
                proteinSynonyms.add(refId);
            }
        }
    }

    private void processDbrefs(Item protein, UniprotEntry entry, List<String> synonymRefIds)
    throws SAXException {
        Map<String, String> dbrefs = entry.getDbrefs();
        if (dbrefs.containsKey("EC")) {
            protein.setAttribute("ecNumber", dbrefs.get("EC"));
        }
        if (dbrefs.containsKey("RefSeq")) {
            String synonym = dbrefs.get("RefSeq");
            String refId = getSynonym(protein.getIdentifier(), "identifier", synonym, "false",
                                      entry.getDatasetRefId());
            synonymRefIds.add(refId);
        }
    }

    // gets the unique identifier and list of identifiers to set
    // loops through each gene entry, assigns refId to protein
    private void processGene(Item protein, UniprotEntry entry)
    throws SAXException {
        String taxonId = entry.getTaxonId();
        // which gene.identifier field has to be unique
        String uniqueIdentifierField = CONFIG.getUniqueIdentifier(taxonId);
        if (uniqueIdentifierField == null) {
            LOG.error("Couldn't process genes for " + taxonId + ", no unique identifier set");
            return;
        }

        // for this organism, set the following gene fields
        Set<String> geneFields = CONFIG.getGeneIdentifierFields(taxonId);

        if (geneFields == null) {
            LOG.error("Couldn't process genes for " + taxonId + ", configuration missing");
            return;
        }

        // just one gene, don't have to worry about gene designations and dbrefs
        if (!entry.hasMultipleGenes()) {
            String geneRefId = createGene(entry, taxonId, geneFields, uniqueIdentifierField);
            if (geneRefId != null) {
                protein.addToCollection("genes", geneRefId);
            }
            return;
        }

        // loop through each gene entry to be processed
        // cloning the gene removes dbrefs without gene designations
        List<UniprotEntry> clonedEntries = entry.cloneGenes();
        Iterator<UniprotEntry> iter = clonedEntries.iterator();
        while (iter.hasNext()) {
            // create a dummy entry and add identifiers for specific gene
            String geneRefId = createGene(iter.next(), taxonId, geneFields, uniqueIdentifierField);
            if (!StringUtil.isEmpty(geneRefId)) {
                protein.addToCollection("genes", geneRefId);
            }
        }
    }

    // creates and stores the gene
    // sets the identifier fields specified in the config file
    // creates synonym
    private String createGene(UniprotEntry entry, String taxonId, Set<String> geneFields,
                              String uniqueIdentifierFieldType)
    throws SAXException {
        List<String> geneSynonyms = new ArrayList();

        String uniqueIdentifierValue = getGeneIdentifier(entry, taxonId, uniqueIdentifierFieldType,
                                                    geneSynonyms, true);
        if (uniqueIdentifierValue == null) {
            return null;
        }
        String geneRefId = genes.get(uniqueIdentifierValue);
        if (geneRefId == null) {
            Item gene = createItem("Gene");
            genes.put(uniqueIdentifierValue, gene.getIdentifier());
            gene.addToCollection("dataSets", entry.getDatasetRefId());
            gene.setAttribute(uniqueIdentifierFieldType, uniqueIdentifierValue);

            // set each identifier
            for (String geneField : geneFields) {
                if (geneField.equals(uniqueIdentifierFieldType)) {
                    // we've already set the key field
                    continue;
                }
                String identifier = getGeneIdentifier(entry, taxonId, geneField, geneSynonyms,
                                                      false);

                if (identifier == null) {
                    LOG.error("Couldn't process gene, no " + geneField);
                    continue;
                }

                /*
                 * if the protein is an isoform, this gene has already been processed so the
                 * identifier will always be a duplicate in this case.
                 */
                if (!entry.isIsoform() && geneIdentifiers.contains(identifier)) {
                    LOG.error("not assigning duplicate identifier:  " + identifier);
                    continue;
                    // if the canonical protein is processed and the gene has a duplicate
                    // identifier, we need to flag so the gene won't be created for the isoform
                    // either.
                }
                geneIdentifiers.add(identifier);
                gene.setAttribute(geneField, identifier);
            }

            // store gene
            try {
                gene.setReference("organism", getOrganism(taxonId));
                store(gene);
            } catch (ObjectStoreException e) {
                throw new SAXException(e);
            }

            // synonyms
            geneRefId = gene.getIdentifier();
            for (String identifier : geneSynonyms) {
                getSynonym(geneRefId, "identifier", identifier, "false", entry.getDatasetRefId());
            }
            getSynonym(geneRefId, "identifier", uniqueIdentifierValue, "false",
                       entry.getDatasetRefId());
        }
        return geneRefId;
    }

    // gets the identifier for a gene from the dbref/names collected from the XML
    // which identifier is chosen depends on the configuration in the uniprot config file
    private String getGeneIdentifier(UniprotEntry entry, String taxonId, String identifierType,
                                     List<String> geneSynonyms, boolean isUniqueIdentifier) {

        String identifierValue = null;
        // how to get the identifier, eg. dbref OR name
        String method = CONFIG.getIdentifierMethod(taxonId, identifierType);
        // what value to use with method, eg. "FlyBase" or "ORF"
        String value = CONFIG.getIdentifierValue(taxonId, identifierType);

        if (method == null || value == null) {
            // use default set in config file, if this organism isn't configured
            method = CONFIG.getIdentifierMethod("default", identifierType);
            value = CONFIG.getIdentifierValue("default", identifierType);
            if (method == null || value == null) {
                throw new RuntimeException("error processing line in config file for organism "
                                           + taxonId);
            }
        }

        if (method.equals("name")) {
            if (entry.getGeneNames() == null || entry.getGeneNames().isEmpty()) {
                LOG.error(" ~~~ No gene names for " + taxonId + ". protein accession:"
                          + entry.getPrimaryAccession());
                return null;
            }
            identifierValue = entry.getGeneNames().get(value);
        } else if (method.equals("dbref")) {
            identifierValue = entry.getDbrefs().get(value);
        } else {
            LOG.error("error processing line in config file for organism " + taxonId);
            return null;
        }
        geneSynonyms.add(identifierValue);
        if (isUniqueIdentifier && taxonId.equals("7227")) {
            identifierValue = resolveGene(taxonId, identifierValue);

            // try again!
            if (identifierValue == null && entry.getGeneNames() != null
                            && !entry.getGeneNames().isEmpty()) {
                Iterator<String> iter = entry.getGeneNames().values().iterator();
                while (iter.hasNext() && identifierValue == null) {
                    identifierValue = resolveGene(taxonId, iter.next());
                }
            }
        }
        return identifierValue;
    }

    private String resolveGene(String taxonId, String identifier) {
        flyResolver = resolverFactory.getIdResolver(false);
        if (flyResolver == null) {
            // no id resolver available, so return the original identifier
            return identifier;
        }
        int resCount = flyResolver.countResolutions(taxonId, identifier);
        if (resCount != 1) {
            LOG.info("RESOLVER: failed to resolve gene to one identifier, ignoring gene: "
                     + identifier + " count: " + resCount + " FBgn: "
                     + flyResolver.resolveId(taxonId, identifier));
            return null;
        }
        return flyResolver.resolveId(taxonId, identifier).iterator().next();
    }

    /* converts the XML into UniProt entry objects.  run once per file */
    private class UniprotHandler extends DefaultHandler
    {
        private UniprotEntry entry;
        private Stack<String> stack = new Stack<String>();
        private String attName = null;
        private StringBuffer attValue = null;
        private String taxonId = null;
        private Set<UniprotEntry> entries;

        /**
         * @param entries empty map to be populated with uniprot entries
         * @param isoforms empty map to be populated with isoforms
         */
        public UniprotHandler(Set<UniprotEntry> entries) {
            this.entries = entries;
{'fraction_non_alphanumeric': 0.09800393008584135, 'fraction_numerical': 0.0015306650118936808, 'mean_word_length': 1.9869022612133942, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '683243', 'n_tokens_mistral': 12524, 'n_tokens_neox': 11728, 'n_words': 3165}