<commit_before>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2011 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.sql.Database;
import org.intermine.xml.full.Item;
import org.intermine.xml.full.ReferenceList;

/**
 * Read Ensembl SNP data directly from MySQL variation database.
 * @author <NAME>
 */
public class EnsemblSnpDbConverter extends BioDBConverter
{
    private static final String DATASET_TITLE = "Ensembl SNP data";
    private static final String DATA_SOURCE_NAME = "Ensembl";
    private final Map<String, Set<String>> pendingSnpConsequences =
        new HashMap<String, Set<String>>();
    private final Map<String, Integer> storedSnpIds = new HashMap<String, Integer>();
    private final Map<String, String> storedSnpItemIdentifiers = new HashMap<String, String>();
    private String dbSnpSourceId;

    // TODO move this to a parser argument
    int taxonId = 9606;
    // Edit to restrict to loading fewer chromosomes
    private static final int MIN_CHROMOSOME = 21;

    private Map<String, String> sources = new HashMap<String, String>();
    private Map<String, String> states = new HashMap<String, String>();
    private Map<String, String> transcripts = new HashMap<String, String>();
    private Map<String, String> noTranscriptConsequences = new HashMap<String, String>();

    private static final Logger LOG = Logger.getLogger(EnsemblSnpDbConverter.class);
    /**
     * Construct a new EnsemblSnpDbConverter.
     * @param database the database to read from
     * @param model the Model used by the object store we will write to with the ItemWriter
     * @param writer an ItemWriter used to handle Items created
     */
    public EnsemblSnpDbConverter(Database database, Model model, ItemWriter writer) {
        super(database, model, writer, DATA_SOURCE_NAME, DATASET_TITLE);
    }

    /**
     * {@inheritDoc}
     */
    public void process() throws Exception {
        // a database has been initialised from properties starting with db.ensembl-snp-db

        Connection connection = getDatabase().getConnection();

        List<String> chrNames = new ArrayList<String>();
        for (int i = MIN_CHROMOSOME; i <= 22; i++) {
            chrNames.add("" + i);
        }
        chrNames.add("X");
        chrNames.add("Y");
        chrNames.add("MT");

        for (String chrName : chrNames) {
            process(connection, chrName);
        }
        connection.close();
    }



    @Override
    public void close() throws Exception {
        LOG.info("CLOSE pendingConsequences.size(): " + pendingSnpConsequences.size());
        LOG.info("CLOSE storeSnpIds.size(): " + storedSnpIds.size());
        for (String rsNumber : pendingSnpConsequences.keySet()) {
            Integer storedSnpId = storedSnpIds.get(rsNumber);
            Set<String> consequenceIdentifiers = pendingSnpConsequences.get(rsNumber);
            ReferenceList col =
                new ReferenceList("consequences", new ArrayList<String>(consequenceIdentifiers));
            store(col, storedSnpId);
        }
    }


    /**
     * {@inheritDoc}
     */
    public void process(Connection connection, String chrName) throws Exception {

        LOG.info("Starting to process chromosome " + chrName);

        ResultSet res = queryVariation(connection, chrName);

        int counter = 0;
        int snpCounter = 0;
        Item currentSnp = null;
        Set<String> seenLocsForSnp = new HashSet<String>();
        String previousRsNumber = null;
        Boolean previousUniqueLocation = true;
        Set<String> consequenceIdentifiers = new HashSet<String>();
        boolean storeSnp = false;
        String currentSnpIdentifier = null;
        Set<String> snpSynonyms = new HashSet<String>();


        while (res.next()) {
            counter++;
            String rsNumber = res.getString("variation_name");
            boolean newSnp = rsNumber.equals(previousRsNumber) ? false : true;

            if (newSnp) {
                // starting a new SNP, store the previous one

                Integer storedSnpId = storedSnpIds.get(previousRsNumber);

                if (storeSnp && storedSnpId == null) {
                    storedSnpId = storeSnp(currentSnp, snpSynonyms);
                    snpCounter++;
                }

                if (previousUniqueLocation) {
                    storeSnpCollections(storedSnpId, consequenceIdentifiers);
                } else {
                    Set<String> snpConsequences = pendingSnpConsequences.get(previousRsNumber);
                    if (snpConsequences == null) {
                        snpConsequences = new HashSet<String>();
                        pendingSnpConsequences.put(previousRsNumber, snpConsequences);
                    }
                    snpConsequences.addAll(consequenceIdentifiers);
                    if (!storedSnpIds.containsKey(previousRsNumber)) {
                        storedSnpIds.put(previousRsNumber, storedSnpId);
                        storedSnpItemIdentifiers.put(previousRsNumber, currentSnp.getIdentifier());
                    }
                }

                // START NEW SNP
                previousRsNumber = rsNumber;
                seenLocsForSnp = new HashSet<String>();
                consequenceIdentifiers = new HashSet<String>();
                snpSynonyms = new HashSet<String>();
                storeSnp = true;

                // map weight is the number of chromosome locations for the SNP, in practice there
                // are sometimes fewer locations than the map_weight indicates
                int mapWeight = res.getInt("map_weight");
                boolean uniqueLocation = (mapWeight == 1) ? true : false;
                previousUniqueLocation = uniqueLocation;

                // if not a unique location and we've seen the SNP before, don't store
                if (!uniqueLocation && pendingSnpConsequences.containsKey(rsNumber)) {
                    LOG.info("Not storing SNP second time: " + rsNumber);
                    storeSnp = false;
                    currentSnpIdentifier = storedSnpItemIdentifiers.get(rsNumber);
                }

                if (!uniqueLocation) {
                    LOG.info("Not a unique location: " + rsNumber + " storeSnp = " + storeSnp);
                }

                if (storeSnp) {
                    currentSnp = createItem("SNP");
                    currentSnp.setAttribute("primaryIdentifier", rsNumber);
                    currentSnp.setReference("organism", getOrganismItem(taxonId));
                    currentSnp.setAttribute("uniqueLocation", "" + uniqueLocation);

                    currentSnpIdentifier = currentSnp.getIdentifier();

                    String alleles = res.getString("allele_string");
                    currentSnp.setAttribute("alleles", alleles);

                    String type = determineType(alleles);
                    if (type != null) {
                        currentSnp.setAttribute("type", type);
                    }

                    // CHROMOSOME AND LOCATION
                    // if SNP is mapped to multiple locations don't set chromosome and
                    // chromosomeLocation references
                    int start = res.getInt("seq_region_start");
                    int end = res.getInt("seq_region_end");
                    int chrStrand = res.getInt("seq_region_strand");

                    int chrStart = Math.min(start, end);
                    int chrEnd = Math.max(start, end);

                    Item loc = createItem("Location");
                    loc.setAttribute("start", "" + chrStart);
                    loc.setAttribute("end", "" + chrEnd);
                    loc.setAttribute("strand", "" + chrStrand);
                    loc.setReference("locatedOn", getChromosome(chrName, taxonId));
                    loc.setReference("feature", currentSnpIdentifier);
                    store(loc);

                    // if mapWeight is 1 there is only one chromosome location, so set shortcuts
                    if (uniqueLocation) {
                        currentSnp.setReference("chromosome", getChromosome(chrName, taxonId));
                        currentSnp.setReference("chromosomeLocation", loc);
                    }
                    seenLocsForSnp.add(chrName + ":" + chrStart);

                    // SOURCE
                    String source = res.getString("s.name");
                    currentSnp.setReference("source", getSourceIdentifier(source));

                    // VALIDATION STATES
                    String validationStatus = res.getString("validation_status");
                    List<String> validationStates = getValidationStateCollection(validationStatus);
                    if (!validationStates.isEmpty()) {
                        currentSnp.setCollection("validations", validationStates);
                    }
                }
            }

            int mapWeight = res.getInt("map_weight");
            boolean uniqueLocation = (mapWeight == 1) ? true : false;

            // we're on the same SNP but maybe a new location

            int start = res.getInt("seq_region_start");
            int end = res.getInt("seq_region_end");
            int strand = res.getInt("seq_region_strand");

            int chrStart = Math.min(start, end);
            int chrEnd = Math.max(start, end);

            if (currentSnp == null) {
                LOG.error("currentSNP is null.  vf.variation_feature_id: "
                        + res.getString("variation_feature_id") + " rsNumber: " + rsNumber
                        + " previousRsNumber: " + previousRsNumber + " storeSnp: " + storeSnp);
            }

            String chrLocStr = chrName + ":" + chrStart;
            if (!seenLocsForSnp.contains(chrLocStr)) {
                seenLocsForSnp.add(chrLocStr);

                // if this location is on a chromosome we want, store it
                Item loc = createItem("Location");
                loc.setAttribute("start", "" + chrStart);
                loc.setAttribute("end", "" + chrEnd);
                loc.setAttribute("strand", "" + strand);
                loc.setReference("feature", currentSnpIdentifier);
                loc.setReference("locatedOn", getChromosome(chrName, taxonId));
                store(loc);
            }


            // CONSEQUENCE TYPES
            // for SNPs without a uniqueLocation there will be different consequences at each one.
            // some consequences will need to stored a the end
            String cdnaStart = res.getString("cdna_start");
            if (StringUtils.isBlank(cdnaStart)) {
                String typeStr = res.getString("vf.consequence_type");
                for (String type : typeStr.split(",")) {
                    consequenceIdentifiers.add(getConsequenceIdentifier(type));
                }
            } else {
                String type = res.getString("tv.consequence_types");
                String peptideAlleles = res.getString("pep_allele_string");
                String transcriptStableId = res.getString("feature_stable_id");

                Item consequenceItem = createItem("Consequence");
                consequenceItem.setAttribute("type", type);
                if (!StringUtils.isBlank(peptideAlleles)) {
                    consequenceItem.setAttribute("peptideAlleles", peptideAlleles);
                }
                if (!StringUtils.isBlank(transcriptStableId)) {
                    consequenceItem.setReference("transcript",
                            getTranscriptIdentifier(transcriptStableId));
                }
                consequenceIdentifiers.add(consequenceItem.getIdentifier());
                store(consequenceItem);
            }

            // SYNONYMS
            // we may see the same synonym on multiple row but the Set will make them unique
            String synonym = res.getString("vs.name");
            if (!StringUtils.isBlank(synonym)) {
                snpSynonyms.add(synonym);
            }
            if (counter % 1000 == 0) {
                LOG.info("Read " + counter + " rows total, stored " + snpCounter + " SNPs. for chr "
                        + chrName);
            }
        }

        if (currentSnp != null) {
            Integer storedSnpId = storeSnp(currentSnp, snpSynonyms);
            if (!storedSnpIds.containsKey(storedSnpId)) {
                storeSnpCollections(storedSnpId, consequenceIdentifiers);
            }
        }
        LOG.info("Finished " + counter + " rows total, stored " + snpCounter + " SNPs for chr "
                + chrName);
    }

    private Integer storeSnp(Item snp, Set<String> synonyms) throws ObjectStoreException {
        for (String synonym : synonyms) {
            createSynonym(snp.getIdentifier(), synonym, true);
        }
        return store(snp);
    }

    private void storeSnpCollections(Integer storedSnpId, Set<String> consequenceIdentifiers)
        throws ObjectStoreException {
        if (!consequenceIdentifiers.isEmpty()) {
            ReferenceList col =
                new ReferenceList("consequences", new ArrayList<String>(consequenceIdentifiers));
            store(col, storedSnpId);
        }
    }

    /**
     * Given an allele string read from the database determine the type of variation, e.g. snp,
     * in-del, etc.  This is a re-implementation of code from the Ensembl perl API, see:
     * http://www.ensembl.org/info/docs/Pdoc/ensembl-variation/
     *     modules/Bio/EnsEMBL/Variation/Utils/Sequence.html#CODE4
     * @param alleleStr the alleles to determine the type for
     * @return a variation class or null if none can be determined
     */
    protected String determineType(String alleleStr) {
        String type = null;

        alleleStr = alleleStr.toUpperCase();
        if (!StringUtils.isBlank(alleleStr)) {
            // snp if e.g. A/C or A|C
            if (alleleStr.matches("^[ACGTN]([\\/\\|\\\\][ACGTN])+$")) {
                type = "snp";
            } else if ("CNV".equals(alleleStr)) {
                type = alleleStr.toLowerCase();
            } else if ("CNV_PROBE".equals(alleleStr)) {
                type = "cnv probe";
            } else if ("HGMD_MUTATION".equals(alleleStr)) {
                type = alleleStr.toLowerCase();
            } else {
                String[] alleles = alleleStr.split("[\\|\\/\\\\]");

                if (alleles.length == 1) {
                    type = "het";
                } else if (alleles.length == 2) {
                    if ((StringUtils.containsOnly(alleles[0], "ACTGN") && "-".equals(alleles[1]))
                            || (StringUtils.containsOnly(alleles[1], "ACTGN")
                                    && "-".equals(alleles[0]))) {
                        type = "in-del";
                    } else if (containsOneOf(alleles[0], "LARGE", "INS", "DEL")
                            || containsOneOf(alleles[1], "LARGE", "INS", "DEL")) {
                        type = "named";
                    } else if ((StringUtils.containsOnly(alleles[0], "ACGT")
                            && alleles[0].length() > 1)
                            || (StringUtils.containsOnly(alleles[1], "ACGT")
                                    && alleles[1].length() > 1)) {
                        // AA/GC 2 alleles
                        type = "substitution";
                    }
                } else if (alleles.length > 2) {
                    if (containsDigit(alleles[0])) {
                        type = "microsat";
                    } else if (anyContainChar(alleles, "-")) {
                        type = "mixed";
                    }
                }
                if (type == null) {
                    LOG.warn("Failed to work out allele type for: " + alleleStr);
                }
            }
        }


        return type;
    }

    private String getSourceIdentifier(String name) throws ObjectStoreException {
        String sourceIdentifier = sources.get(name);
        if (sourceIdentifier == null) {
            Item source = createItem("Source");
            source.setAttribute("name", name);
            store(source);
            sourceIdentifier = source.getIdentifier();
            sources.put(name, sourceIdentifier);
        }
        return sourceIdentifier;
    }

    private String getTranscriptIdentifier(String transcriptStableId) throws ObjectStoreException {
        String transcriptIdentifier = transcripts.get(transcriptStableId);
        if (transcriptIdentifier == null) {
            Item transcript = createItem("Transcript");
            transcript.setAttribute("primaryIdentifier", transcriptStableId);
            store(transcript);
            transcriptIdentifier = transcript.getIdentifier();
            transcripts.put(transcriptStableId, transcriptIdentifier);
        }
        return transcriptIdentifier;
    }

    private List<String> getValidationStateCollection(String input) throws ObjectStoreException {
        List<String> stateIdentifiers = new ArrayList<String>();
        if (!StringUtils.isBlank(input)) {
            for (String state : input.split(",")) {
                stateIdentifiers.add(getStateIdentifier(state));
            }
        }
        return stateIdentifiers;
    }

    private String getStateIdentifier(String name) throws ObjectStoreException {
        String stateIdentifier = states.get(name);
        if (stateIdentifier == null) {
            Item state = createItem("ValidationState");
            state.setAttribute("name", name);
            store(state);
            stateIdentifier = state.getIdentifier();
            states.put(name, stateIdentifier);
        }
        return stateIdentifier;
    }

    private String getConsequenceIdentifier(String type) throws ObjectStoreException {
        String consequenceIdentifier = noTranscriptConsequences.get(type);
        if (consequenceIdentifier == null) {
            Item consequence = createItem("Consequence");
            consequence.setAttribute("type", type);
            store(consequence);
            consequenceIdentifier = consequence.getIdentifier();
            noTranscriptConsequences.put(type, consequenceIdentifier);
        }
        return consequenceIdentifier;
    }

    private ResultSet queryVariation(Connection connection, String chrName)
        throws SQLException {

        String query = "SELECT vf.variation_feature_id, vf.variation_name, vf.allele_string, "
            + " sr.name,"
            + " vf.map_weight, vf.seq_region_start, vf.seq_region_end, vf.seq_region_strand, "
            + " s.name,"
            + " vf.validation_status,"
            + " vf.consequence_type,"
            + " tv.cdna_start,tv.consequence_types,tv.pep_allele_string,tv.feature_stable_id,"
            + " vs.name"
            + " FROM seq_region sr, source s, variation_feature vf "
            + " LEFT JOIN (transcript_variation tv)"
            + " ON (vf.variation_feature_id = tv.variation_feature_id"
            + "     AND tv.cdna_start is not null)"
            + " LEFT JOIN (variation_synonym vs)"
            + " ON (vf.variation_id = vs.variation_id"
            + "     AND vs.source_id = " + getDbSnpSourceId(connection) + ")"
            + " WHERE vf.seq_region_id = sr.seq_region_id"
            + " AND vf.source_id = s.source_id"
            + " AND sr.name = '" + chrName + "'"
            + " ORDER BY vf.variation_id";

        LOG.warn(query);
        System.out.println(query);

        Statement stmt = connection.createStatement();
        ResultSet res = stmt.executeQuery(query);
        return res;
    }

    private String getDbSnpSourceId(Connection connection) throws SQLException {
        if (dbSnpSourceId == null) {
            String query = "SELECT source_id FROM source WHERE name = \"dbSNP\"";
            Statement stmt = connection.createStatement();
            ResultSet res = stmt.executeQuery(query);
            res.next();
            dbSnpSourceId = res.getString("source_id");
            if (dbSnpSourceId == null) {
                throw new RuntimeException("Failed to retrieve source_id for dbSNP source");
            }
        }
        return dbSnpSourceId;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String getDataSetTitle(int taxonId) {
        return DATASET_TITLE;
    }

    private boolean containsOneOf(String target, String... substrings) {
        for (String substring : substrings) {
            if (target.contains(substring)) {
                return true;
            }
        }
        return false;
    }

    private boolean anyContainChar(String[] targets, String substring) {
        for (String target : targets) {
            if (target.contains(substring)) {
                return true;
            }
        }
        return false;
    }

    private boolean containsDigit(String target) {
        for (int i = 0; i < target.length(); i++) {
            if (Character.isDigit(target.charAt(i))) {
                return true;
            }
        }
        return false;
    }
}
<commit_msg>Fix number of chromosomes loaded from Ensembl SNP.<commit_after>package org.intermine.bio.dataconversion;

/*
 * Copyright (C) 2002-2011 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.intermine.dataconversion.ItemWriter;
import org.intermine.metadata.Model;
import org.intermine.objectstore.ObjectStoreException;
import org.intermine.sql.Database;
import org.intermine.xml.full.Item;
import org.intermine.xml.full.ReferenceList;

/**
 * Read Ensembl SNP data directly from MySQL variation database.
 * @author <NAME>
 */
public class EnsemblSnpDbConverter extends BioDBConverter
{
    private static final String DATASET_TITLE = "Ensembl SNP data";
    private static final String DATA_SOURCE_NAME = "Ensembl";
    private final Map<String, Set<String>> pendingSnpConsequences =
        new HashMap<String, Set<String>>();
    private final Map<String, Integer> storedSnpIds = new HashMap<String, Integer>();
    private final Map<String, String> storedSnpItemIdentifiers = new HashMap<String, String>();
    private String dbSnpSourceId;

    // TODO move this to a parser argument
    int taxonId = 9606;
    // Edit to restrict to loading fewer chromosomes
    private static final int MIN_CHROMOSOME = 1;

    private Map<String, String> sources = new HashMap<String, String>();
    private Map<String, String> states = new HashMap<String, String>();
    private Map<String, String> transcripts = new HashMap<String, String>();
    private Map<String, String> noTranscriptConsequences = new HashMap<String, String>();

    private static final Logger LOG = Logger.getLogger(EnsemblSnpDbConverter.class);
    /**
     * Construct a new EnsemblSnpDbConverter.
     * @param database the database to read from
     * @param model the Model used by the object store we will write to with the ItemWriter
     * @param writer an ItemWriter used to handle Items created
     */
    public EnsemblSnpDbConverter(Database database, Model model, ItemWriter writer) {
        super(database, model, writer, DATA_SOURCE_NAME, DATASET_TITLE);
    }

    /**
     * {@inheritDoc}
     */
    public void process() throws Exception {
        // a database has been initialised from properties starting with db.ensembl-snp-db

        Connection connection = getDatabase().getConnection();

        List<String> chrNames = new ArrayList<String>();
        for (int i = MIN_CHROMOSOME; i <= 22; i++) {
            chrNames.add("" + i);
        }
        chrNames.add("X");
        chrNames.add("Y");
        chrNames.add("MT");

        for (String chrName : chrNames) {
            process(connection, chrName);
        }
        connection.close();
    }



    @Override
    public void close() throws Exception {
        LOG.info("CLOSE pendingConsequences.size(): " + pendingSnpConsequences.size());
        LOG.info("CLOSE storeSnpIds.size(): " + storedSnpIds.size());
        for (String rsNumber : pendingSnpConsequences.keySet()) {
            Integer storedSnpId = storedSnpIds.get(rsNumber);
            Set<String> consequenceIdentifiers = pendingSnpConsequences.get(rsNumber);
            ReferenceList col =
                new ReferenceList("consequences", new ArrayList<String>(consequenceIdentifiers));
            store(col, storedSnpId);
        }
    }


    /**
     * {@inheritDoc}
     */
    public void process(Connection connection, String chrName) throws Exception {

        LOG.info("Starting to process chromosome " + chrName);

        ResultSet res = queryVariation(connection, chrName);

        int counter = 0;
        int snpCounter = 0;
        Item currentSnp = null;
        Set<String> seenLocsForSnp = new HashSet<String>();
        String previousRsNumber = null;
        Boolean previousUniqueLocation = true;
        Set<String> consequenceIdentifiers = new HashSet<String>();
        boolean storeSnp = false;
        String currentSnpIdentifier = null;
        Set<String> snpSynonyms = new HashSet<String>();


        while (res.next()) {
            counter++;
            String rsNumber = res.getString("variation_name");
            boolean newSnp = rsNumber.equals(previousRsNumber) ? false : true;

            if (newSnp) {
                // starting a new SNP, store the previous one

                Integer storedSnpId = storedSnpIds.get(previousRsNumber);

                if (storeSnp && storedSnpId == null) {
                    storedSnpId = storeSnp(currentSnp, snpSynonyms);
                    snpCounter++;
                }

                if (previousUniqueLocation) {
                    storeSnpCollections(storedSnpId, consequenceIdentifiers);
                } else {
                    Set<String> snpConsequences = pendingSnpConsequences.get(previousRsNumber);
                    if (snpConsequences == null) {
                        snpConsequences = new HashSet<String>();
                        pendingSnpConsequences.put(previousRsNumber, snpConsequences);
                    }
                    snpConsequences.addAll(consequenceIdentifiers);
                    if (!storedSnpIds.containsKey(previousRsNumber)) {
                        storedSnpIds.put(previousRsNumber, storedSnpId);
                        storedSnpItemIdentifiers.put(previousRsNumber, currentSnp.getIdentifier());
                    }
                }

                // START NEW SNP
                previousRsNumber = rsNumber;
                seenLocsForSnp = new HashSet<String>();
                consequenceIdentifiers = new HashSet<String>();
                snpSynonyms = new HashSet<String>();
                storeSnp = true;

                // map weight is the number of chromosome locations for the SNP, in practice there
                // are sometimes fewer locations than the map_weight indicates
                int mapWeight = res.getInt("map_weight");
                boolean uniqueLocation = (mapWeight == 1) ? true : false;
                previousUniqueLocation = uniqueLocation;

                // if not a unique location and we've seen the SNP before, don't store
                if (!uniqueLocation && pendingSnpConsequences.containsKey(rsNumber)) {
                    LOG.info("Not storing SNP second time: " + rsNumber);
                    storeSnp = false;
                    currentSnpIdentifier = storedSnpItemIdentifiers.get(rsNumber);
                }

                if (!uniqueLocation) {
                    LOG.info("Not a unique location: " + rsNumber + " storeSnp = " + storeSnp);
                }

                if (storeSnp) {
                    currentSnp = createItem("SNP");
                    currentSnp.setAttribute("primaryIdentifier", rsNumber);
                    currentSnp.setReference("organism", getOrganismItem(taxonId));
                    currentSnp.setAttribute("uniqueLocation", "" + uniqueLocation);

                    currentSnpIdentifier = currentSnp.getIdentifier();

                    String alleles = res.getString("allele_string");
                    currentSnp.setAttribute("alleles", alleles);

                    String type = determineType(alleles);
                    if (type != null) {
                        currentSnp.setAttribute("type", type);
                    }

                    // CHROMOSOME AND LOCATION
                    // if SNP is mapped to multiple locations don't set chromosome and
                    // chromosomeLocation references
                    int start = res.getInt("seq_region_start");
                    int end = res.getInt("seq_region_end");
                    int chrStrand = res.getInt("seq_region_strand");

                    int chrStart = Math.min(start, end);
                    int chrEnd = Math.max(start, end);

                    Item loc = createItem("Location");
                    loc.setAttribute("start", "" + chrStart);
                    loc.setAttribute("end", "" + chrEnd);
                    loc.setAttribute("strand", "" + chrStrand);
                    loc.setReference("locatedOn", getChromosome(chrName, taxonId));
                    loc.setReference("feature", currentSnpIdentifier);
                    store(loc);

                    // if mapWeight is 1 there is only one chromosome location, so set shortcuts
                    if (uniqueLocation) {
                        currentSnp.setReference("chromosome", getChromosome(chrName, taxonId));
                        currentSnp.setReference("chromosomeLocation", loc);
                    }
                    seenLocsForSnp.add(chrName + ":" + chrStart);

                    // SOURCE
                    String source = res.getString("s.name");
                    currentSnp.setReference("source", getSourceIdentifier(source));

                    // VALIDATION STATES
                    String validationStatus = res.getString("validation_status");
                    List<String> validationStates = getValidationStateCollection(validationStatus);
                    if (!validationStates.isEmpty()) {
                        currentSnp.setCollection("validations", validationStates);
                    }
                }
            }

            int mapWeight = res.getInt("map_weight");
            boolean uniqueLocation = (mapWeight == 1) ? true : false;

            // we're on the same SNP but maybe a new location

            int start = res.getInt("seq_region_start");
            int end = res.getInt("seq_region_end");
            int strand = res.getInt("seq_region_strand");

            int chrStart = Math.min(start, end);
            int chrEnd = Math.max(start, end);

            if (currentSnp == null) {
                LOG.error("currentSNP is null.  vf.variation_feature_id: "
                        + res.getString("variation_feature_id") + " rsNumber: " + rsNumber
                        + " previousRsNumber: " + previousRsNumber + " storeSnp: " + storeSnp);
            }

            String chrLocStr = chrName + ":" + chrStart;
            if (!seenLocsForSnp.contains(chrLocStr)) {
                seenLocsForSnp.add(chrLocStr);

                // if this location is on a chromosome we want, store it
                Item loc = createItem("Location");
                loc.setAttribute("start", "" + chrStart);
                loc.setAttribute("end", "" + chrEnd);
                loc.setAttribute("strand", "" + strand);
                loc.setReference("feature", currentSnpIdentifier);
                loc.setReference("locatedOn", getChromosome(chrName, taxonId));
                store(loc);
            }


            // CONSEQUENCE TYPES
            // for SNPs without a uniqueLocation there will be different consequences at each one.
            // some consequences will need to stored a the end
            String cdnaStart = res.getString("cdna_start");
            if (StringUtils.isBlank(cdnaStart)) {
                String typeStr = res.getString("vf.consequence_type");
                for (String type : typeStr.split(",")) {
                    consequenceIdentifiers.add(getConsequenceIdentifier(type));
                }
            } else {
                String type = res.getString("tv.consequence_types");
                String peptideAlleles = res.getString("pep_allele_string");
                String transcriptStableId = res.getString("feature_stable_id");

                Item consequenceItem = createItem("Consequence");
                consequenceItem.setAttribute("type", type);
                if (!StringUtils.isBlank(peptideAlleles)) {
                    consequenceItem.setAttribute("peptideAlleles", peptideAlleles);
                }
                if (!StringUtils.isBlank(transcriptStableId)) {
                    consequenceItem.setReference("transcript",
                            getTranscriptIdentifier(transcriptStableId));
                }
                consequenceIdentifiers.add(consequenceItem.getIdentifier());
                store(consequenceItem);
            }

            // SYNONYMS
            // we may see the same synonym on multiple row but the Set will make them unique
            String synonym = res.getString("vs.name");
            if (!StringUtils.isBlank(synonym)) {
                snpSynonyms.add(synonym);
            }
            if (counter % 1000 == 0) {
                LOG.info("Read " + counter + " rows total, stored " + snpCounter + " SNPs. for chr "
                        + chrName);
            }
        }

        if (currentSnp != null) {
            Integer storedSnpId = storeSnp(currentSnp, snpSynonyms);
            if (!storedSnpIds.containsKey(storedSnpId)) {
                storeSnpCollections(storedSnpId, consequenceIdentifiers);
            }
        }
        LOG.info("Finished " + counter + " rows total, stored " + snpCounter + " SNPs for chr "
                + chrName);
    }

    private Integer storeSnp(Item snp, Set<String> synonyms) throws ObjectStoreException {
        for (String synonym : synonyms) {
            createSynonym(snp.getIdentifier(), synonym, true);
        }
        return store(snp);
    }

    private void storeSnpCollections(Integer storedSnpId, Set<String> consequenceIdentifiers)
        throws ObjectStoreException {
        if (!consequenceIdentifiers.isEmpty()) {
            ReferenceList col =
                new ReferenceList("consequences", new ArrayList<String>(consequenceIdentifiers));
            store(col, storedSnpId);
        }
    }

    /**
     * Given an allele string read from the database determine the type of variation, e.g. snp,
     * in-del, etc.  This is a re-implementation of code from the Ensembl perl API, see:
     * http://www.ensembl.org/info/docs/Pdoc/ensembl-variation/
     *     modules/Bio/EnsEMBL/Variation/Utils/Sequence.html#CODE4
     * @param alleleStr the alleles to determine the type for
     * @return a variation class or null if none can be determined
     */
    protected String determineType(String alleleStr) {
        String type = null;

        alleleStr = alleleStr.toUpperCase();
        if (!StringUtils.isBlank(alleleStr)) {
            // snp if e.g. A/C or A|C
            if (alleleStr.matches("^[ACGTN]([\\/\\|\\\\][ACGTN])+$")) {
                type = "snp";
            } else if ("CNV".equals(alleleStr)) {
                type = alleleStr.toLowerCase();
            } else if ("CNV_PROBE".equals(alleleStr)) {
                type = "cnv probe";
            } else if ("HGMD_MUTATION".equals(alleleStr)) {
                type = alleleStr.toLowerCase();
            } else {
                String[] alleles = alleleStr.split("[\\|\\/\\\\]");

                if (alleles.length == 1) {
                    type = "het";
                } else if (alleles.length == 2) {
                    if ((StringUtils.containsOnly(alleles[0], "ACTGN") && "-".equals(alleles[1]))
                            || (StringUtils.containsOnly(alleles[1], "ACTGN")
                                    && "-".equals(alleles[0]))) {
                        type = "in-del";
                    } else if (containsOneOf(alleles[0], "LARGE", "INS", "DEL")
                            || containsOneOf(alleles[1], "LARGE", "INS", "DEL")) {
                        type = "named";
                    } else if ((StringUtils.containsOnly(alleles[0], "ACGT")
                            && alleles[0].length() > 1)
                            || (StringUtils.containsOnly(alleles[1], "ACGT")
                                    && alleles[1].length() > 1)) {
                        // AA/GC 2 alleles
                        type = "substitution";
                    }
                } else if (alleles.length > 2) {
                    if (containsDigit(alleles[0])) {
                        type = "microsat";
                    } else if (anyContainChar(alleles, "-")) {
                        type = "mixed";
                    }
                }
                if (type == null) {
                    LOG.warn("Failed to work out allele type for: " + alleleStr);
                }
            }
        }


        return type;
    }

    private String getSourceIdentifier(String name) throws ObjectStoreException {
        String sourceIdentifier = sources.get(name);
        if (sourceIdentifier == null) {
            Item source = createItem("Source");
            source.setAttribute("name", name);
            store(source);
            sourceIdentifier = source.getIdentifier();
            sources.put(name, sourceIdentifier);
        }
        return sourceIdentifier;
    }

    private String getTranscriptIdentifier(String transcriptStableId) throws ObjectStoreException {
        String transcriptIdentifier = transcripts.get(transcriptStableId);
        if (transcriptIdentifier == null) {
            Item transcript = createItem("Transcript");
            transcript.setAttribute("primaryIdentifier", transcriptStableId);
            store(transcript);
            transcriptIdentifier = transcript.getIdentifier();
            transcripts.put(transcriptStableId, transcriptIdentifier);
        }
        return transcriptIdentifier;
    }

    private List<String> getValidationStateCollection(String input) throws ObjectStoreException {
        List<String> stateIdentifiers = new ArrayList<String>();
        if (!StringUtils.isBlank(input)) {
            for (String state : input.split(",")) {
                stateIdentifiers.add(getStateIdentifier(state));
            }
        }
        return stateIdentifiers;
    }

    private String getStateIdentifier(String name) throws ObjectStoreException {
        String stateIdentifier = states.get(name);
        if (stateIdentifier == null) {
            Item state = createItem("ValidationState");
            state.setAttribute("name", name);
            store(state);
            stateIdentifier = state.getIdentifier();
            states.put(name, stateIdentifier);
        }
        return stateIdentifier;
    }

    private String getConsequenceIdentifier(String type) throws ObjectStoreException {
        String consequenceIdentifier = noTranscriptConsequences.get(type);
        if (consequenceIdentifier == null) {
            Item consequence = createItem("Consequence");
            consequence.setAttribute("type", type);
            store(consequence);
            consequenceIdentifier = consequence.getIdentifier();
            noTranscriptConsequences.put(type, consequenceIdentifier);
        }
        return consequenceIdentifier;
    }

    private ResultSet queryVariation(Connection connection, String chrName)
        throws SQLException {

        String query = "SELECT vf.variation_feature_id, vf.variation_name, vf.allele_string, "
            + " sr.name,"
            + " vf.map_weight, vf.seq_region_start, vf.seq_region_end, vf.seq_region_strand, "
            + " s.name,"
            + " vf.validation_status,"
            + " vf.consequence_type,"
            + " tv.cdna_start,tv.consequence_types,tv.pep_allele_string,tv.feature_stable_id,"
            + " vs.name"
            + " FROM seq_region sr, source s, variation_feature vf "
            + " LEFT JOIN (transcript_variation tv)"
            + " ON (vf.variation_feature_id = tv.variation_feature_id"
            + "     AND tv.cdna_start is not null)"
            + " LEFT JOIN (variation_synonym vs)"
            + " ON (vf.variation_id = vs.variation_id"
            + "     AND vs.source_id = " + getDbSnpSourceId(connection) + ")"
            + " WHERE vf.seq_region_id = sr.seq_region_id"
            + " AND vf.source_id = s.source_id"
            + " AND sr.name = '" + chrName + "'"
            + " ORDER BY vf.variation_id";

        LOG.warn(query);
        System.out.println(query);

        Statement stmt = connection.createStatement();
        ResultSet res = stmt.executeQuery(query);
        return res;
    }

    private String getDbSnpSourceId(Connection connection) throws SQLException {
        if (dbSnpSourceId == null) {
            String query = "SELECT source_id FROM source WHERE name = \"dbSNP\"";
            Statement stmt = connection.createStatement();
            ResultSet res = stmt.executeQuery(query);
            res.next();
            dbSnpSourceId = res.getString("source_id");
            if (dbSnpSourceId == null) {
                throw new RuntimeException("Failed to retrieve source_id for dbSNP source");
            }
        }
        return dbSnpSourceId;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public String getDataSetTitle(int taxonId) {
        return DATASET_TITLE;
    }

    private boolean containsOneOf(String target, String... substrings) {
        for (String substring : substrings) {
            if (target.contains(substring)) {
                return true;
            }
        }
        return false;
    }

    private boolean anyContainChar(String[] targets, String substring) {
        for (String target : targets) {
            if (target.contains(substring)) {
                return true;
            }
        }
        return false;
    }

    private boolean containsDigit(String target) {
        for (int i = 0; i < target.length(); i++) {
            if (Character.isDigit(target.charAt(i))) {
                return true;
            }
        }
        return false;
    }
}

{'fraction_non_alphanumeric': 0.10064087891966125, 'fraction_numerical': 0.0020828564888990615, 'mean_word_length': 1.9082739798974906, 'pattern_counts': {'":': 4, '<': 85, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5456738', 'n_tokens_mistral': 11647, 'n_tokens_neox': 10898, 'n_words': 2611}