<commit_before>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2010 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.sql.Database;
import org.intermine.xml.full.Item;

/**
 * Read Ensembl SNP data directly from MySQL variarion database.
 * @author <NAME>
 */
public class EnsemblSnpDbConverter extends BioDBConverter
{
    private static final String DATASET_TITLE = "Ensembl SNP data";
    private static final String DATA_SOURCE_NAME = "Ensembl";

    // TODO move this to a parser argument
    int taxonId = 9606;


    private Map<String, String> sources = new HashMap<String, String>();
    private Map<String, String> states = new HashMap<String, String>();
    private Map<String, String> transcripts = new HashMap<String, String>();
    private Map<String, String> noTranscriptConsequences = new HashMap<String, String>();

    private static final Logger LOG = Logger.getLogger(EnsemblSnpDbConverter.class);
    /**
     * Construct a new EnsemblSnpDbConverter.
     * @param database the database to read from
     * @param model the Model used by the object store we will write to with the ItemWriter
     * @param writer an ItemWriter used to handle Items created
     */
    public EnsemblSnpDbConverter(Database database, Model model, ItemWriter writer) {
        super(database, model, writer, DATA_SOURCE_NAME, DATASET_TITLE);
    }


    /**
     * {@inheritDoc}
     */
    public void process() throws Exception {
        // a database has been initialised from properties starting with db.ensembl-snp-db

        Connection connection = getDatabase().getConnection();

        Set<String> chrNames = new HashSet<String>();
        //int MIN_CHROMOSOME = 1;
        int MIN_CHROMOSOME = 20;
        for (int i = MIN_CHROMOSOME; i <= 22; i++) {
            chrNames.add("" + i);
        }
        chrNames.add("X");
        chrNames.add("Y");

        process(connection, chrNames);
        connection.close();
    }

    /**
     * {@inheritDoc}
     */
    public void process(Connection connection, Set<String> chrNames) throws Exception {

        ResultSet res = queryVariation(connection, chrNames);

        int counter = 0;
        int snpCounter = 0;
        Item currentSnp = null;
        Set<String> seenLocsForSnp = new HashSet<String>();
        String currentRsNumber = null;
        Set<String> consequenceIdentifiers = new HashSet<String>();

        while (res.next()) {
            counter++;
            String rsNumber = res.getString("variation_name");
            String chrName = res.getString("sr.name");

            if (rsNumber.equals(currentRsNumber)) {
                int start = res.getInt("seq_region_start");
                int end = res.getInt("seq_region_end");
                int strand = res.getInt("seq_region_strand");

                int chrStart = Math.min(start, end);
                int chrEnd = Math.max(start, end);

                String chrLocStr = chrName + ":" + chrStart;
                if (!seenLocsForSnp.contains(chrLocStr)) {
                    seenLocsForSnp.add(chrLocStr);

                    // if this location is on a chromosome we want, store it
                    Item loc = createItem("Location");
                    loc.setAttribute("start", "" + chrStart);
                    loc.setAttribute("end", "" + chrEnd);
                    loc.setAttribute("strand", "" + strand);
                    loc.setReference("feature", currentSnp);
                    loc.setReference("locatedOn", getChromosome(chrName, taxonId));
                    store(loc);
                    LOG.info("Found duplicate locations for SNP: " + rsNumber + " this: "
                            + chrLocStr + " previous " + seenLocsForSnp);
                }
            }
            if (!rsNumber.equals(currentRsNumber)) {<commit_msg>Remove logging.<commit_after>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2010 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.sql.Database;
import org.intermine.xml.full.Item;

/**
 * Read Ensembl SNP data directly from MySQL variarion database.
 * @author <NAME>
 */
public class EnsemblSnpDbConverter extends BioDBConverter
{
    private static final String DATASET_TITLE = "Ensembl SNP data";
    private static final String DATA_SOURCE_NAME = "Ensembl";

    // TODO move this to a parser argument
    int taxonId = 9606;


    private Map<String, String> sources = new HashMap<String, String>();
    private Map<String, String> states = new HashMap<String, String>();
    private Map<String, String> transcripts = new HashMap<String, String>();
    private Map<String, String> noTranscriptConsequences = new HashMap<String, String>();

    private static final Logger LOG = Logger.getLogger(EnsemblSnpDbConverter.class);
    /**
     * Construct a new EnsemblSnpDbConverter.
     * @param database the database to read from
     * @param model the Model used by the object store we will write to with the ItemWriter
     * @param writer an ItemWriter used to handle Items created
     */
    public EnsemblSnpDbConverter(Database database, Model model, ItemWriter writer) {
        super(database, model, writer, DATA_SOURCE_NAME, DATASET_TITLE);
    }


    /**
     * {@inheritDoc}
     */
    public void process() throws Exception {
        // a database has been initialised from properties starting with db.ensembl-snp-db

        Connection connection = getDatabase().getConnection();

        Set<String> chrNames = new HashSet<String>();
        //int MIN_CHROMOSOME = 1;
        int MIN_CHROMOSOME = 20;
        for (int i = MIN_CHROMOSOME; i <= 22; i++) {
            chrNames.add("" + i);
        }
        chrNames.add("X");
        chrNames.add("Y");

        process(connection, chrNames);
        connection.close();
    }

    /**
     * {@inheritDoc}
     */
    public void process(Connection connection, Set<String> chrNames) throws Exception {

        ResultSet res = queryVariation(connection, chrNames);

        int counter = 0;
        int snpCounter = 0;
        Item currentSnp = null;
        Set<String> seenLocsForSnp = new HashSet<String>();
        String currentRsNumber = null;
        Set<String> consequenceIdentifiers = new HashSet<String>();

        while (res.next()) {
            counter++;
            String rsNumber = res.getString("variation_name");
            String chrName = res.getString("sr.name");

            if (rsNumber.equals(currentRsNumber)) {
                int start = res.getInt("seq_region_start");
                int end = res.getInt("seq_region_end");
                int strand = res.getInt("seq_region_strand");

                int chrStart = Math.min(start, end);
                int chrEnd = Math.max(start, end);

                String chrLocStr = chrName + ":" + chrStart;
                if (!seenLocsForSnp.contains(chrLocStr)) {
                    seenLocsForSnp.add(chrLocStr);

                    // if this location is on a chromosome we want, store it
                    Item loc = createItem("Location");
                    loc.setAttribute("start", "" + chrStart);
                    loc.setAttribute("end", "" + chrEnd);
                    loc.setAttribute("strand", "" + strand);
                    loc.setReference("feature", currentSnp);
                    loc.setReference("locatedOn", getChromosome(chrName, taxonId));
                    store(loc);
                }
            }
            if (!rsNumber.equals(currentRsNumber)) {
{'fraction_non_alphanumeric': 0.104661586732407, 'fraction_numerical': 0.0044822949350067235, 'mean_word_length': 2.5529458598726116, 'pattern_counts': {'":': 2, '<': 37, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3321433', 'n_tokens_mistral': 2526, 'n_tokens_neox': 2391, 'n_words': 681}