<commit_before>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2008 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import org.intermine.dataconversion.*;
import org.intermine.metadata.Model;
import org.intermine.xml.full.*;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.InterMineException;
import org.intermine.util.XmlUtil;
import org.apache.log4j.Logger;

import java.util.*;
import java.io.IOException;
import java.io.FileInputStream;
import java.io.FileNotFoundException;

/**
 * Convert Ensembl data in fulldata Item format conforming to a source OWL definition
 * to fulldata Item format conforming to InterMine OWL definition.
 *
 * @author <NAME>
 * @author <NAME>
 * @author <NAME>
 */
public class EnsemblDataTranslator extends DataTranslator
{
    protected static final String PATH_NAME_SPACE = "http://www.intermine.org/model/ensembl#";

    protected static final ItemPath TRANSCRIPT_VIA_TRANSLATION =
            new ItemPath("(transcript <- translation.transcript)", PATH_NAME_SPACE);

    protected static final ItemPath EXON_VIA_EXON_TRANSCRIPT =
            new ItemPath("(exon <- exon_transcript.exon)", PATH_NAME_SPACE);

    protected static final ItemPath SEQ_REGION_VIA_DNA =
            new ItemPath("(seq_region <- dna.seq_region)", PATH_NAME_SPACE);

    protected static final ItemPath MARKER_VIA_MARKER_SYNONYM =
            new ItemPath("(marker <- marker_synonym.marker)", PATH_NAME_SPACE);

    protected static final Logger LOG = Logger.getLogger(EnsemblDataTranslator.class);

    private Map seqIdMap = new HashMap();
    private Map itemId2SynMap = new HashMap();

    private Map markerSynonymMap = new HashMap();

    private String srcNs;

    //Holds all the information from the Ensembl Config file - i.e. datasources & sets
    private EnsemblConfig config;

    /**
     * @param srcItemReader Our source item provider.
     * @param mergeSpec The contents of our mapping spec.
     * @param srcModel The data model we are translating items from.
     * @param tgtModel The data model we are translating items into.
     * @param propsFileName File with set of Properties that are organism specific.
     * @param orgAbbrev A suitably short acronym to identify which ogransim to process; 'HS' = Human
     * @deprecated Have to get the translator to look for its props file now...
     */
    public EnsemblDataTranslator(ItemReader srcItemReader,
                                 Properties mergeSpec,
                                 Model srcModel,
                                 Model tgtModel,
                                 String propsFileName,
                                 String orgAbbrev) {

        super(srcItemReader, mergeSpec, srcModel, tgtModel);

        Properties ensemblProps;
        try {
            ensemblProps = getEnsemblProperties(propsFileName);
        } catch (ObjectStoreException e) {
            throw new RuntimeException(e);
        }

        config = new EnsemblConfig(ensemblProps, orgAbbrev);
    }

    /**
     * @param srcItemReader Our source item provider.
     * @param mergeSpec The contents of our mapping spec.
     * @param srcModel The data model we are translating items from.
     * @param tgtModel The data model we are translating items into.
     * @param ensemblProps Properties that are organism specific.
     * @param orgAbbrev A suitably short acronym to identify which ogransim to process; 'HS' = Human
     */
    public EnsemblDataTranslator(ItemReader srcItemReader,
                                 Properties mergeSpec,
                                 Model srcModel,
                                 Model tgtModel,
                                 Properties ensemblProps,
                                 String orgAbbrev) {
        super(srcItemReader, mergeSpec, srcModel, tgtModel);

        config = new EnsemblConfig(ensemblProps, orgAbbrev);
    }

    /**
     * {@inheritDoc}
     */
    public void translate(ItemWriter tgtItemWriter)
            throws ObjectStoreException, InterMineException {

        tgtItemWriter.store(ItemHelper.convert(config.getOrganism()));
        tgtItemWriter.store(ItemHelper.convert(config.getEnsemblDataSet()));

        super.translate(tgtItemWriter);

        for (Iterator dsIt = config.getDataSrcItemIterator(); dsIt.hasNext(); ) {

            tgtItemWriter.store(ItemHelper.convert((Item) dsIt.next()));
        }

        for (Iterator seqSynIt = itemId2SynMap.values().iterator(); seqSynIt.hasNext();) {

            tgtItemWriter.store(ItemHelper.convert((Item) seqSynIt.next()));
        }
    }

    /**
     * {@inheritDoc}
     */
    protected Collection translateItem(Item srcItem)
            throws ObjectStoreException, InterMineException {
        Collection result = new ArrayList();
        srcNs = XmlUtil.getNamespaceFromURI(srcItem.getClassName());
        String srcItemClassName = XmlUtil.getFragmentFromURI(srcItem.getClassName());

        Collection translated = super.translateItem(srcItem);
        if (translated != null) {
            for (Iterator i = translated.iterator(); i.hasNext();) {
                boolean storeTgtItem = true;
                Item tgtItem = (Item) i.next();
                // store tgtItem first as other objects will point to it
                result.add(tgtItem);
                if ("dna".equals(srcItemClassName)) {

                    if (config.doStoreDna()) {

                        if (srcItem.hasAttribute("sequence")) {
                            int seqLen = srcItem.getAttribute("sequence").getValue().length();
                            tgtItem.setAttribute("length", Integer.toString(seqLen));
                        }
                    } else {
                        storeTgtItem = false;
                    }
                } else if ("karyotype".equals(srcItemClassName)) {

                    translateKaryotype(srcItem, tgtItem, result);
                } else if ("exon".equals(srcItemClassName)) {

                    //Skip this exon if it points to invalid transcripts!
                    // if (!isExonToBeKept(srcItem, true)) {
//                         LOG.debug("isExonToBeKept false for Exon:" + srcItem.getIdentifier());
//                         return result;
//                     }

                    tgtItem.addReference(config.getOrganismRef());
                    Item stableId = getStableId("exon", srcItem.getIdentifier(), srcNs);
                    // <- exon_stable_id.exon
                    if (stableId != null) {
                        moveField(stableId, tgtItem, "stable_id", "primaryIdentifier");
                    }
                    addReferencedItem(tgtItem, config.getEnsemblDataSet(),
                            "evidence", true, "", false);
                    Item location = createLocation(srcItem, tgtItem, true); // seq_region
                    // seq_region.coord-sys
                    result.add(location);

                } else if ("gene".equals(srcItemClassName)) {
                    tgtItem.addReference(config.getOrganismRef());
                    addReferencedItem(tgtItem, config.getEnsemblDataSet(),
                            "evidence", true, "", false);
                    Item comment = createComment(srcItem, tgtItem);
                    if (comment != null) {
                        result.add(comment);
                    }
                    Item location = createLocation(srcItem, tgtItem, true); // seq_region
                    // seq_region.coord-sys
                    result.add(location);
                    if (config.createAnalysisResult) {
                        Item anaResult = createAnalysisResult(srcItem, tgtItem); // analysis
                        result.add(anaResult);
                    }

                    // the default gene identifier should be its stable id (or identifier if none)
                    //i.e. for anoph they are the same, but for human they are differant
                    //note: the primaryIdentifier is effecivly what we choose as a primary accession
                    // and thus may differ from what we want to assign as the identifier...
                    result.addAll(setGeneSynonyms(srcItem, tgtItem, srcNs));

                } else if ("transcript".equals(srcItemClassName)) {

                    translateTranscript(srcItem, tgtItem, result, srcNs);

                } else if ("translation".equals(srcItemClassName)) {
                    tgtItem.addReference(config.getOrganismRef());

                    // if no identifier set the identifier as name (primary key)
                    if (!tgtItem.hasAttribute("primaryIdentifier")) {
                        Item stableId = getStableId("translation", srcItem.getIdentifier(), srcNs);
                        // <- transcript_stable_id.transcript
                        if (stableId != null) {
                            moveField(stableId, tgtItem, "stable_id", "primaryIdentifier");
                        } else {
                            tgtItem.addAttribute(
                                    new Attribute("primaryIdentifier", srcItem.getIdentifier()));
                        }
                    }

                // stable_ids become syonyms, need ensembl DataSet as evidence
                } else if (srcItemClassName.endsWith("_stable_id")) {

                    //check to see if the exons point to valid transcripts before making a synonym.
                   //  if (srcItemClassName.startsWith("exon")) {
//                         //Skip this exon if it points to invalid transcripts!
//                         if (!isExonToBeKept(srcItem, false)) {
//                             return result;
//                         }
//                     }

                    tgtItem.addToCollection("evidence", config.getEnsemblDataSet());
                    tgtItem.addReference(config.getEnsemblDataSrcRef());
                    tgtItem.addAttribute(new Attribute("type", "identifier"));
                } else if ("repeat_feature".equals(srcItemClassName)) {

                    translateRepeatFeature(srcItem, tgtItem, result);
                    // repeat_consensus

                } else if ("marker".equals(srcItemClassName)) {
                    tgtItem.addReference(config.getOrganismRef());

                    //is there an identifier set?
                    if (!srcItem.hasAttribute("primaryIdentifier")) {

                        //ok try and look up any related marker_synonym items...

                    }

                    addReferencedItem(tgtItem,
                            config.getEnsemblDataSet(), "evidence", true, "", false);
                    Set locations = createLocations(srcItem, tgtItem, srcNs);
                    // <- marker_feature.marker<commit_msg>Corrected identifier/primaryIdentifier bug, line of code was possibly never called.<commit_after>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2008 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import org.intermine.dataconversion.*;
import org.intermine.metadata.Model;
import org.intermine.xml.full.*;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.InterMineException;
import org.intermine.util.XmlUtil;
import org.apache.log4j.Logger;

import java.util.*;
import java.io.IOException;
import java.io.FileInputStream;
import java.io.FileNotFoundException;

/**
 * Convert Ensembl data in fulldata Item format conforming to a source OWL definition
 * to fulldata Item format conforming to InterMine OWL definition.
 *
 * @author <NAME>
 * @author <NAME>
 * @author <NAME>
 */
public class EnsemblDataTranslator extends DataTranslator
{
    protected static final String PATH_NAME_SPACE = "http://www.intermine.org/model/ensembl#";

    protected static final ItemPath TRANSCRIPT_VIA_TRANSLATION =
            new ItemPath("(transcript <- translation.transcript)", PATH_NAME_SPACE);

    protected static final ItemPath EXON_VIA_EXON_TRANSCRIPT =
            new ItemPath("(exon <- exon_transcript.exon)", PATH_NAME_SPACE);

    protected static final ItemPath SEQ_REGION_VIA_DNA =
            new ItemPath("(seq_region <- dna.seq_region)", PATH_NAME_SPACE);

    protected static final ItemPath MARKER_VIA_MARKER_SYNONYM =
            new ItemPath("(marker <- marker_synonym.marker)", PATH_NAME_SPACE);

    protected static final Logger LOG = Logger.getLogger(EnsemblDataTranslator.class);

    private Map seqIdMap = new HashMap();
    private Map itemId2SynMap = new HashMap();

    private Map markerSynonymMap = new HashMap();

    private String srcNs;

    //Holds all the information from the Ensembl Config file - i.e. datasources & sets
    private EnsemblConfig config;

    /**
     * @param srcItemReader Our source item provider.
     * @param mergeSpec The contents of our mapping spec.
     * @param srcModel The data model we are translating items from.
     * @param tgtModel The data model we are translating items into.
     * @param propsFileName File with set of Properties that are organism specific.
     * @param orgAbbrev A suitably short acronym to identify which ogransim to process; 'HS' = Human
     * @deprecated Have to get the translator to look for its props file now...
     */
    public EnsemblDataTranslator(ItemReader srcItemReader,
                                 Properties mergeSpec,
                                 Model srcModel,
                                 Model tgtModel,
                                 String propsFileName,
                                 String orgAbbrev) {

        super(srcItemReader, mergeSpec, srcModel, tgtModel);

        Properties ensemblProps;
        try {
            ensemblProps = getEnsemblProperties(propsFileName);
        } catch (ObjectStoreException e) {
            throw new RuntimeException(e);
        }

        config = new EnsemblConfig(ensemblProps, orgAbbrev);
    }

    /**
     * @param srcItemReader Our source item provider.
     * @param mergeSpec The contents of our mapping spec.
     * @param srcModel The data model we are translating items from.
     * @param tgtModel The data model we are translating items into.
     * @param ensemblProps Properties that are organism specific.
     * @param orgAbbrev A suitably short acronym to identify which ogransim to process; 'HS' = Human
     */
    public EnsemblDataTranslator(ItemReader srcItemReader,
                                 Properties mergeSpec,
                                 Model srcModel,
                                 Model tgtModel,
                                 Properties ensemblProps,
                                 String orgAbbrev) {
        super(srcItemReader, mergeSpec, srcModel, tgtModel);

        config = new EnsemblConfig(ensemblProps, orgAbbrev);
    }

    /**
     * {@inheritDoc}
     */
    public void translate(ItemWriter tgtItemWriter)
            throws ObjectStoreException, InterMineException {

        tgtItemWriter.store(ItemHelper.convert(config.getOrganism()));
        tgtItemWriter.store(ItemHelper.convert(config.getEnsemblDataSet()));

        super.translate(tgtItemWriter);

        for (Iterator dsIt = config.getDataSrcItemIterator(); dsIt.hasNext(); ) {

            tgtItemWriter.store(ItemHelper.convert((Item) dsIt.next()));
        }

        for (Iterator seqSynIt = itemId2SynMap.values().iterator(); seqSynIt.hasNext();) {

            tgtItemWriter.store(ItemHelper.convert((Item) seqSynIt.next()));
        }
    }

    /**
     * {@inheritDoc}
     */
    protected Collection translateItem(Item srcItem)
            throws ObjectStoreException, InterMineException {
        Collection result = new ArrayList();
        srcNs = XmlUtil.getNamespaceFromURI(srcItem.getClassName());
        String srcItemClassName = XmlUtil.getFragmentFromURI(srcItem.getClassName());

        Collection translated = super.translateItem(srcItem);
        if (translated != null) {
            for (Iterator i = translated.iterator(); i.hasNext();) {
                boolean storeTgtItem = true;
                Item tgtItem = (Item) i.next();
                // store tgtItem first as other objects will point to it
                result.add(tgtItem);
                if ("dna".equals(srcItemClassName)) {

                    if (config.doStoreDna()) {

                        if (srcItem.hasAttribute("sequence")) {
                            int seqLen = srcItem.getAttribute("sequence").getValue().length();
                            tgtItem.setAttribute("length", Integer.toString(seqLen));
                        }
                    } else {
                        storeTgtItem = false;
                    }
                } else if ("karyotype".equals(srcItemClassName)) {

                    translateKaryotype(srcItem, tgtItem, result);
                } else if ("exon".equals(srcItemClassName)) {

                    //Skip this exon if it points to invalid transcripts!
                    // if (!isExonToBeKept(srcItem, true)) {
//                         LOG.debug("isExonToBeKept false for Exon:" + srcItem.getIdentifier());
//                         return result;
//                     }

                    tgtItem.addReference(config.getOrganismRef());
                    Item stableId = getStableId("exon", srcItem.getIdentifier(), srcNs);
                    // <- exon_stable_id.exon
                    if (stableId != null) {
                        moveField(stableId, tgtItem, "stable_id", "primaryIdentifier");
                    }
                    addReferencedItem(tgtItem, config.getEnsemblDataSet(),
                            "evidence", true, "", false);
                    Item location = createLocation(srcItem, tgtItem, true); // seq_region
                    // seq_region.coord-sys
                    result.add(location);

                } else if ("gene".equals(srcItemClassName)) {
                    tgtItem.addReference(config.getOrganismRef());
                    addReferencedItem(tgtItem, config.getEnsemblDataSet(),
                            "evidence", true, "", false);
                    Item comment = createComment(srcItem, tgtItem);
                    if (comment != null) {
                        result.add(comment);
                    }
                    Item location = createLocation(srcItem, tgtItem, true); // seq_region
                    // seq_region.coord-sys
                    result.add(location);
                    if (config.createAnalysisResult) {
                        Item anaResult = createAnalysisResult(srcItem, tgtItem); // analysis
                        result.add(anaResult);
                    }

                    // the default gene identifier should be its stable id (or identifier if none)
                    //i.e. for anoph they are the same, but for human they are differant
                    //note: the primaryIdentifier is effecivly what we choose as a primary accession
                    // and thus may differ from what we want to assign as the identifier...
                    result.addAll(setGeneSynonyms(srcItem, tgtItem, srcNs));

                } else if ("transcript".equals(srcItemClassName)) {

                    translateTranscript(srcItem, tgtItem, result, srcNs);

                } else if ("translation".equals(srcItemClassName)) {
                    tgtItem.addReference(config.getOrganismRef());

                    // if no identifier set the identifier as name (primary key)
                    if (!tgtItem.hasAttribute("primaryIdentifier")) {
                        Item stableId = getStableId("translation", srcItem.getIdentifier(), srcNs);
                        // <- transcript_stable_id.transcript
                        if (stableId != null) {
                            moveField(stableId, tgtItem, "stable_id", "primaryIdentifier");
                        } else {
                            tgtItem.addAttribute(
                                    new Attribute("primaryIdentifier", srcItem.getIdentifier()));
                        }
                    }

                // stable_ids become syonyms, need ensembl DataSet as evidence
                } else if (srcItemClassName.endsWith("_stable_id")) {

                    //check to see if the exons point to valid transcripts before making a synonym.
                   //  if (srcItemClassName.startsWith("exon")) {
//                         //Skip this exon if it points to invalid transcripts!
//                         if (!isExonToBeKept(srcItem, false)) {
//                             return result;
//                         }
//                     }

                    tgtItem.addToCollection("evidence", config.getEnsemblDataSet());
                    tgtItem.addReference(config.getEnsemblDataSrcRef());
                    tgtItem.addAttribute(new Attribute("type", "identifier"));
                } else if ("repeat_feature".equals(srcItemClassName)) {

                    translateRepeatFeature(srcItem, tgtItem, result);
                    // repeat_consensus

                } else if ("marker".equals(srcItemClassName)) {
                    tgtItem.addReference(config.getOrganismRef());

                    //is there an identifier set?
                    if (!srcItem.hasAttribute("identifier")) {

                        //ok try and look up any related marker_synonym items...

                    }

                    addReferencedItem(tgtItem,
                            config.getEnsemblDataSet(), "evidence", true, "", false);
                    Set locations = createLocations(srcItem, tgtItem, srcNs);
                    // <- marker_feature.marker
{'fraction_non_alphanumeric': 0.08805909643709743, 'fraction_numerical': 0.0009909463537678483, 'mean_word_length': 1.8146551724137931, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3877235', 'n_tokens_mistral': 5655, 'n_tokens_neox': 5273, 'n_words': 1426}