<commit_before>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2008 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.FileConverter;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.util.FormattedTextParser;
import org.intermine.xml.full.Item;

/**
 *
 * @author <NAME>
 */
public class Drosophila2ProbeConverter extends FileConverter
{
    protected static final Logger LOG = Logger.getLogger(Drosophila2ProbeConverter.class);

    protected Item dataSource, dataSet, org;
    protected Map<String, Item> bioentities = new HashMap<String, Item>();
    protected IdResolverFactory resolverFactory;
    private static final String TAXON_ID = "7227";
    private Map<String, Item> synonyms = new HashMap<String, Item>();
    private Map<String, String> chromosomes = new HashMap<String, String>();
    private Map<String, ProbeHolder> holders;
    private static final Set<String> CHROMOSOMES = new HashSet<String>();

    static {
        CHROMOSOMES.add("2L");
        CHROMOSOMES.add("2R");
        CHROMOSOMES.add("3L");
        CHROMOSOMES.add("3R");
        CHROMOSOMES.add("4");
        CHROMOSOMES.add("U");
        CHROMOSOMES.add("X");
    }

    /**
     * Constructor
     * @param writer the ItemWriter used to handle the resultant items
     * @param model the data model
     * @throws ObjectStoreException if an error occurs in storing
     */
    public Drosophila2ProbeConverter(ItemWriter writer, Model model)
        throws ObjectStoreException {
        super(writer, model);

        dataSource = createItem("DataSource");
        dataSource.setAttribute("name", "Affymetrix");
        store(dataSource);

        org = createItem("Organism");
        org.setAttribute("taxonId", TAXON_ID);
        store(org);

        // only construct factory here so can be replaced by mock factory in tests
        resolverFactory = new FlyBaseIdResolverFactory();
    }


    /**
     * Read each line from flat file.
     *
     * {@inheritDoc}
     */
    public void process(Reader reader) throws Exception {

        Iterator<String[]> lineIter = FormattedTextParser.parseTabDelimitedReader(reader);
        List<Item> delayedItems = new ArrayList<Item>();
        holders = new HashMap<String, ProbeHolder>();
        boolean hasDataset = false;

        while (lineIter.hasNext()) {
            String[] line = lineIter.next();
            if (!hasDataset) {
                createDataset(line[0]);
                hasDataset = true;
            }

            String probesetIdentifier = line[1];
            String transcriptIdentifier = line[2];
            String fbgn = line[3];
            String chromosomeIdentifier = line[4];
            String startString = line[5];
            String endString = line[6];
            String strand = line[7];
            String hitCount = line[8];
            String[] hitStrings = hitCount.split(" ");
            boolean hasHits = false;
            Integer hits = new Integer(0);
            try {
                hits = new Integer(hitStrings[3]);
                // probe has to have >= 13 hits to be processed
                if (hits.compareTo(new Integer(13)) >= 0) {
                    hasHits = true;
                }
            } catch (Exception e) {
                // don't process this data
            }

            if (CHROMOSOMES.contains(chromosomeIdentifier) && hasHits) {
                String chromosomeRefId = createChromosome(chromosomeIdentifier);
                Item gene = createGene(fbgn, delayedItems);
                if (gene != null) {
                    Item transcript = createTranscript(transcriptIdentifier, gene.getIdentifier(),
                                                       delayedItems);
                    ProbeHolder holder = getHolder(probesetIdentifier, transcript.getIdentifier(),
                                                   gene.getIdentifier(), hits);
                    try {
                        Integer start = new Integer(startString);
                        Integer end = new Integer(endString);
                        holder.setLocation(chromosomeRefId, start, end, strand);
                    } catch (NumberFormatException e) {
                        throw new RuntimeException("bad start/end values");
                    }
                }
            }
        }
        for (ProbeHolder holder : holders.values()) {
            storeProbeSet(holder, delayedItems);<commit_msg>load probesets from all chromosomes.  Refs #1815.  probesets now have a locations collection so we can store all chromosome locations.  also, removing these chromosomes didn't have much of an impact on the number of duplicates that were created.<commit_after>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2008 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.FileConverter;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.util.FormattedTextParser;
import org.intermine.xml.full.Item;

/**
 *
 * @author <NAME>
 */
public class Drosophila2ProbeConverter extends FileConverter
{
    protected static final Logger LOG = Logger.getLogger(Drosophila2ProbeConverter.class);

    protected Item dataSource, dataSet, org;
    protected Map<String, Item> bioentities = new HashMap<String, Item>();
    protected IdResolverFactory resolverFactory;
    private static final String TAXON_ID = "7227";
    private Map<String, Item> synonyms = new HashMap<String, Item>();
    private Map<String, String> chromosomes = new HashMap<String, String>();
    private Map<String, ProbeHolder> holders;
//    private static final Set<String> CHROMOSOMES = new HashSet<String>();
//
//    static {
//        CHROMOSOMES.add("2L");
//        CHROMOSOMES.add("2R");
//        CHROMOSOMES.add("3L");
//        CHROMOSOMES.add("3R");
//        CHROMOSOMES.add("4");
//        CHROMOSOMES.add("U");
//        CHROMOSOMES.add("X");
//    }

    /**
     * Constructor
     * @param writer the ItemWriter used to handle the resultant items
     * @param model the data model
     * @throws ObjectStoreException if an error occurs in storing
     */
    public Drosophila2ProbeConverter(ItemWriter writer, Model model)
        throws ObjectStoreException {
        super(writer, model);

        dataSource = createItem("DataSource");
        dataSource.setAttribute("name", "Affymetrix");
        store(dataSource);

        org = createItem("Organism");
        org.setAttribute("taxonId", TAXON_ID);
        store(org);

        // only construct factory here so can be replaced by mock factory in tests
        resolverFactory = new FlyBaseIdResolverFactory();
    }


    /**
     * Read each line from flat file.
     *
     * {@inheritDoc}
     */
    public void process(Reader reader) throws Exception {

        Iterator<String[]> lineIter = FormattedTextParser.parseTabDelimitedReader(reader);
        List<Item> delayedItems = new ArrayList<Item>();
        holders = new HashMap<String, ProbeHolder>();
        boolean hasDataset = false;

        while (lineIter.hasNext()) {
            String[] line = lineIter.next();
            if (!hasDataset) {
                createDataset(line[0]);
                hasDataset = true;
            }

            String probesetIdentifier = line[1];
            String transcriptIdentifier = line[2];
            String fbgn = line[3];
            String chromosomeIdentifier = line[4];
            String startString = line[5];
            String endString = line[6];
            String strand = line[7];
            String hitCount = line[8];
            String[] hitStrings = hitCount.split(" ");
            boolean hasHits = false;
            Integer hits = new Integer(0);
            try {
                hits = new Integer(hitStrings[3]);
                // probe has to have >= 13 hits to be processed
                if (hits.compareTo(new Integer(13)) >= 0) {
                    hasHits = true;
                }
            } catch (Exception e) {
                // don't process this data
            }

//            if (CHROMOSOMES.contains(chromosomeIdentifier) && hasHits) {
            if (hasHits) {
                String chromosomeRefId = createChromosome(chromosomeIdentifier);
                Item gene = createGene(fbgn, delayedItems);
                if (gene != null) {
                    Item transcript = createTranscript(transcriptIdentifier, gene.getIdentifier(),
                                                       delayedItems);
                    ProbeHolder holder = getHolder(probesetIdentifier, transcript.getIdentifier(),
                                                   gene.getIdentifier(), hits);
                    try {
                        Integer start = new Integer(startString);
                        Integer end = new Integer(endString);
                        holder.setLocation(chromosomeRefId, start, end, strand);
                    } catch (NumberFormatException e) {
                        throw new RuntimeException("bad start/end values");
                    }
                }
            }
        }
        for (ProbeHolder holder : holders.values()) {
            storeProbeSet(holder, delayedItems);
{'fraction_non_alphanumeric': 0.09722355653761168, 'fraction_numerical': 0.0074935152272072245, 'mean_word_length': 2.1795968234575445, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '145451', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2631, 'n_words': 726}