<commit_before>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2005 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Arrays;
import java.util.TreeSet;

import org.intermine.metadata.Model;
import org.intermine.xml.full.Attribute;
import org.intermine.xml.full.Item;
import org.intermine.xml.full.ReferenceList;
import org.intermine.util.XmlUtil;

import org.intermine.bio.io.gff3.GFF3Record;

import org.apache.log4j.Logger;

/**
 * A converter/retriever for FlyBase GFF3 files.
 *
 * @author <NAME>
 */

public class FlyBaseGFF3RecordHandler extends GFF3RecordHandler
{
    Map references;
    private static final Logger LOG = Logger.getLogger(FlyBaseGFF3RecordHandler.class);
    private String tgtNs;
    private Set pseudogeneIds = new HashSet();
    private Map otherOrganismItems = new HashMap();
    private Map cdsStarts = new HashMap(), cdsEnds = new HashMap();
    private Set cdss = new HashSet();

    // items that need extra processing that can only be done after all other GFF features have
    // been read
    private Collection finalItems = new ArrayList();

    /**
     * Create a new FlyBaseGFF3RecordHandler for the given target model.
     * @param tgtModel the model for which items will be created
     */
    public FlyBaseGFF3RecordHandler(Model tgtModel) {
        super(tgtModel);
        tgtNs = tgtModel.getNameSpace().toString();

        // create a map of classname to reference name for parent references
        // this will add the parents of any SimpleRelations from getParents() to the
        // given collection
        references = new HashMap();
        references.put("Enhancer", "gene");
        references.put("Exon", "transcripts");
        references.put("InsertionSite", "genes");
        references.put("Intron", "transcripts");
        // release 4.0 gff3 is inconsistent with parents of RNAs
        // Gene.transcripts collection set in post-processing
        references.put("MRNA", "gene");
        references.put("NcRNA", "gene");
        references.put("SnRNA", "gene");
        references.put("SnoRNA", "gene");
        references.put("TRNA", "gene");
        references.put("PointMutation", "genes");
        references.put("PolyASite", "processedTranscripts");
        // Region is inherited by loads of things, causes conflicts
        //references.put("Region", "gene");
        references.put("RegulatoryRegion", "gene");
        references.put("SequenceVariant", "genes");
        references.put("FivePrimeUTR", "MRNAs");
        references.put("ThreePrimeUTR", "MRNAs");
        references.put("CDS", "MRNAs");
    }

    /**
     * This method does the following transformations.
     *
     * For all
     * @see GFF3RecordHandler#process(GFF3Record)
     */
    public void process(GFF3Record record) {
        Item feature = getFeature();

        if (!feature.hasAttribute("curated")) {
            feature.addAttribute(new Attribute("curated", "true"));
        }

        String clsName = XmlUtil.getFragmentFromURI(feature.getClassName());

        if ("RegulatoryRegion".equals(clsName)) {
            feature.setClassName(tgtNs + "TFmodule");
            clsName = "TFmodule";
        }

        if ("Protein".equals(clsName)) {
            // for v4.3 CDS record changed to proteins
            feature.setClassName(tgtNs + "Translation");
            clsName = "Translation";
        }

        if ("CDS".equals(clsName)) {
            // keep CDSs until the end, need to be combined into one CDS per CDS Name per
            // transcript (Parent).  This needs to have a location with the lowest start and
            // highest end value of all CDS parts.
            String cdsId = feature.getAttribute("identifier").getValue();
            String cdsName = feature.getAttribute("symbol").getValue();
            String transcriptId = null;
            Iterator transIter = record.getParents().iterator();
            try {
                transcriptId = (String) transIter.next();
            } catch (Exception e) {
                throw new RuntimeException("no parent found for CDS: " + cdsId);
            }
            if (transIter.hasNext()) {
                throw new RuntimeException("multiple parents found for CDS: " + cdsId);
            }

            CDSHolder holder = new CDSHolder(cdsName, transcriptId,
                                             getLocation().getAttribute("strand").getValue());
            cdss.add(holder);

            // TODO does reference to transcript get set?
            Integer start = new Integer(getLocation().getAttribute("start").getValue());
            Integer end = new Integer(getLocation().getAttribute("end").getValue());
            Set starts = (Set) cdsStarts.get(holder);
            if (starts == null) {
                starts = new TreeSet();
                cdsStarts.put(holder, starts);
            }
            starts.add(start);
<commit_msg>added error message<commit_after>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2005 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Arrays;
import java.util.TreeSet;

import org.intermine.metadata.Model;
import org.intermine.xml.full.Attribute;
import org.intermine.xml.full.Item;
import org.intermine.xml.full.ReferenceList;
import org.intermine.util.XmlUtil;

import org.intermine.bio.io.gff3.GFF3Record;

import org.apache.log4j.Logger;

/**
 * A converter/retriever for FlyBase GFF3 files.
 *
 * @author <NAME>
 */

public class FlyBaseGFF3RecordHandler extends GFF3RecordHandler
{
    Map references;
    private static final Logger LOG = Logger.getLogger(FlyBaseGFF3RecordHandler.class);
    private String tgtNs;
    private Set pseudogeneIds = new HashSet();
    private Map otherOrganismItems = new HashMap();
    private Map cdsStarts = new HashMap(), cdsEnds = new HashMap();
    private Set cdss = new HashSet();

    // items that need extra processing that can only be done after all other GFF features have
    // been read
    private Collection finalItems = new ArrayList();

    /**
     * Create a new FlyBaseGFF3RecordHandler for the given target model.
     * @param tgtModel the model for which items will be created
     */
    public FlyBaseGFF3RecordHandler(Model tgtModel) {
        super(tgtModel);
        tgtNs = tgtModel.getNameSpace().toString();

        // create a map of classname to reference name for parent references
        // this will add the parents of any SimpleRelations from getParents() to the
        // given collection
        references = new HashMap();
        references.put("Enhancer", "gene");
        references.put("Exon", "transcripts");
        references.put("InsertionSite", "genes");
        references.put("Intron", "transcripts");
        // release 4.0 gff3 is inconsistent with parents of RNAs
        // Gene.transcripts collection set in post-processing
        references.put("MRNA", "gene");
        references.put("NcRNA", "gene");
        references.put("SnRNA", "gene");
        references.put("SnoRNA", "gene");
        references.put("TRNA", "gene");
        references.put("PointMutation", "genes");
        references.put("PolyASite", "processedTranscripts");
        // Region is inherited by loads of things, causes conflicts
        //references.put("Region", "gene");
        references.put("RegulatoryRegion", "gene");
        references.put("SequenceVariant", "genes");
        references.put("FivePrimeUTR", "MRNAs");
        references.put("ThreePrimeUTR", "MRNAs");
        references.put("CDS", "MRNAs");
    }

    /**
     * This method does the following transformations.
     *
     * For all
     * @see GFF3RecordHandler#process(GFF3Record)
     */
    public void process(GFF3Record record) {
        Item feature = getFeature();

        if (!feature.hasAttribute("curated")) {
            feature.addAttribute(new Attribute("curated", "true"));
        }

        String clsName = XmlUtil.getFragmentFromURI(feature.getClassName());

        if ("RegulatoryRegion".equals(clsName)) {
            feature.setClassName(tgtNs + "TFmodule");
            clsName = "TFmodule";
        }

        if ("Protein".equals(clsName)) {
            // for v4.3 CDS record changed to proteins
            feature.setClassName(tgtNs + "Translation");
            clsName = "Translation";
        }

        if ("CDS".equals(clsName)) {
            // keep CDSs until the end, need to be combined into one CDS per CDS Name per
            // transcript (Parent).  This needs to have a location with the lowest start and
            // highest end value of all CDS parts.
            String cdsId = feature.getAttribute("identifier").getValue();
            String cdsName = feature.getAttribute("symbol").getValue();
            String transcriptId = null;
            Iterator transIter;
            try {
                transIter = record.getParents().iterator();
            } catch (Exception e) {
                throw new RuntimeException("error getting parents for CDS: " + cdsId, e);
            }
            try {
                transcriptId = (String) transIter.next();
            } catch (Exception e) {
                throw new RuntimeException("no parent found for CDS: " + cdsId, e);
            }
            if (transIter.hasNext()) {
                throw new RuntimeException("multiple parents found for CDS: " + cdsId);
            }

            CDSHolder holder = new CDSHolder(cdsName, transcriptId,
                                             getLocation().getAttribute("strand").getValue());
            cdss.add(holder);

            // TODO does reference to transcript get set?
            Integer start = new Integer(getLocation().getAttribute("start").getValue());
            Integer end = new Integer(getLocation().getAttribute("end").getValue());
            Set starts = (Set) cdsStarts.get(holder);
            if (starts == null) {
                starts = new TreeSet();
                cdsStarts.put(holder, starts);
            }
            starts.add(start);

{'fraction_non_alphanumeric': 0.11121190130624092, 'fraction_numerical': 0.0045355587808417995, 'mean_word_length': 2.524616368286445, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1121500', 'n_tokens_mistral': 3038, 'n_tokens_neox': 2947, 'n_words': 841}