<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2018] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import csv
import json


BLAT_GENOMES = {
    'anopheles_gambiae',
    'arabidopsis_thaliana',
    'bombyx_mori',
    'caenorhabditis_elegans',
    'dictyostelium_discoideum',
    'drosophila_melanogaster',
    'homo_sapiens',
    'mus_musculus',
    'plasmodium_falciparum',
    'rattus_norvegicus'
    'saccharomyces_cerevisiae',
    'schizosaccharomyces_pombe',
}


def domain_url(division):
    """Given E! division, returns E!/E! Genomes url."""

    if division == 'Ensembl':
        subdomain = 'ensembl.org'
    elif division == 'EnsemblPlants':
        subdomain = 'plants.ensembl.org'
    elif division == 'EnsemblMetazoa':
        subdomain = 'metazoa.ensembl.org'
    elif division == 'EnsemblBacteria':
        subdomain = 'bacteria.ensembl.org'
    elif division == 'EnsemblFungi':
        subdomain = 'fungi.ensembl.org'
    elif division == 'EnsemblProtists':
        subdomain = 'protists.ensembl.org'
    else:
        subdomain = None
    return subdomain


def reconcile_taxids(taxid):
    """
    Sometimes Ensembl taxid and ENA/Expert Database taxid do not match,
    so to reconcile the differences, taxids in the ensembl_assembly table
    are overriden to match other data.
    """

    taxid = str(taxid)
    if taxid == '284812':  # Ensembl assembly for Schizosaccharomyces pombe
        return 4896  # Pombase and ENA xrefs for Schizosaccharomyces pombe
    return int(taxid)


def load_genomes(handle):
    return set(line.strip() for line in handle)


def parse(handle, example_locations):
    """
    This parses the TSV file for assembly information and produces an array
    of the data to write to the database.
    """

    reader = csv.reader(handle, delimiter='\t')
    assembly = dict(reader)

    example_location = {'chromosome': None, 'start': None, 'end': None}
    key = assembly['species.url'].lower()
    if key in example_locations:
        example_location = example_locations[key]

    url = assembly['species.url'].lower()
    is_mapped = int(url in BLAT_GENOMES)
    return [
        assembly['assembly.default'],
        assembly['assembly.name'],
        assembly.get('assembly.accession', None),
        assembly.get('assembly.ucsc_alias', None),
        assembly.get('species.common_name', None),
        reconcile_taxids(assembly['species.taxonomy_id']),
        url,
        assembly['species.division'],
        domain_url(assembly['species.division']),
        example_location['chromosome'],
        example_location['start'],
        example_location['end'],
        is_mapped,
    ]


def write(handle, example_handle, output):

    example_locations = json.load(example_handle)
    data = parse(handle, example_locations)
    csv.writer(output).writerow(data)<commit_msg>Not ready for proper example handling<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2018] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import csv
import json


BLAT_GENOMES = {
    'anopheles_gambiae',
    'arabidopsis_thaliana',
    'bombyx_mori',
    'caenorhabditis_elegans',
    'dictyostelium_discoideum',
    'drosophila_melanogaster',
    'homo_sapiens',
    'mus_musculus',
    'plasmodium_falciparum',
    'rattus_norvegicus'
    'saccharomyces_cerevisiae',
    'schizosaccharomyces_pombe',
}


def domain_url(division):
    """Given E! division, returns E!/E! Genomes url."""

    if division == 'Ensembl':
        subdomain = 'ensembl.org'
    elif division == 'EnsemblPlants':
        subdomain = 'plants.ensembl.org'
    elif division == 'EnsemblMetazoa':
        subdomain = 'metazoa.ensembl.org'
    elif division == 'EnsemblBacteria':
        subdomain = 'bacteria.ensembl.org'
    elif division == 'EnsemblFungi':
        subdomain = 'fungi.ensembl.org'
    elif division == 'EnsemblProtists':
        subdomain = 'protists.ensembl.org'
    else:
        subdomain = None
    return subdomain


EXAMPLE_LOCATIONS = {
    'homo_sapiens': {
        'chromosome': 'X',
        'start': 73819307,
        'end': 73856333,
    },
    'mus_musculus': {
        'chromosome': 1,
        'start': 86351908,
        'end': 86352200,
    },
    'danio_rerio': {
        'chromosome': 9,
        'start': 7633910,
        'end': 7634210,
    },
    'bos_taurus': {
        'chromosome': 15,
        'start': 82197673,
        'end': 82197837,
    },
    'rattus_norvegicus': {
        'chromosome': 'X',
        'start': 118277628,
        'end': 118277850,
    },
    'felis_catus': {
        'chromosome': 'X',
        'start': 18058223,
        'end': 18058546,
    },
    'macaca_mulatta': {
        'chromosome': 1,
        'start': 146238837,
        'end': 146238946,
    },
    'pan_troglodytes': {
        'chromosome': 11,
        'start': 78369004,
        'end': 78369219,
    },
    'canis_familiaris': {
        'chromosome': 19,
        'start': 22006909,
        'end': 22007119,
    },
    'gallus_gallus': {
        'chromosome': 9,
        'start': 15676031,
        'end': 15676160,
    },
    'xenopus_tropicalis': {
        'chromosome': 'NC_006839',
        'start': 11649,
        'end': 11717,
    },
    'saccharomyces_cerevisiae': {
        'chromosome': 'XII',
        'start': 856709,
        'end': 856919,
    },
    'schizosaccharomyces_pombe': {
        'chromosome': 'I',
        'start': 540951,
        'end': 544327,
    },
    'caenorhabditis_elegans': {
        'chromosome': 'III',
        'start': 11467363,
        'end': 11467705,
    },
    'drosophila_melanogaster': {
        'chromosome': '3R',
        'start': 7474331,
        'end': 7475217,
    },
    'bombyx_mori': {
        'chromosome': 'scaf16',
        'start': 6180018,
        'end': 6180422,
    },
    'anopheles_gambiae': {
        'chromosome': '2R',
        'start': 34644956,
        'end': 34645131,
    },
    'dictyostelium_discoideum': {
        'chromosome': 2,
        'start': 7874546,
        'end': 7876498,
    },
    'plasmodium_falciparum': {
        'chromosome': 13,
        'start': 2796339,
        'end': 2798488,
    },
    'arabidopsis_thaliana': {
        'chromosome': 2,
        'start': 18819643,
        'end': 18822629,
    }
}


def reconcile_taxids(taxid):
    """
    Sometimes Ensembl taxid and ENA/Expert Database taxid do not match,
    so to reconcile the differences, taxids in the ensembl_assembly table
    are overriden to match other data.
    """

    taxid = str(taxid)
    if taxid == '284812':  # Ensembl assembly for Schizosaccharomyces pombe
        return 4896  # Pombase and ENA xrefs for Schizosaccharomyces pombe
    return int(taxid)


def load_genomes(handle):
    return set(line.strip() for line in handle)


def parse(handle, example_locations):
    """
    This parses the TSV file for assembly information and produces an array
    of the data to write to the database.
    """

    reader = csv.reader(handle, delimiter='\t')
    assembly = dict(reader)

    example_location = {'chromosome': None, 'start': None, 'end': None}
    key = assembly['species.url'].lower()
    if key in example_locations:
        example_location = example_locations[key]

    url = assembly['species.url'].lower()
    is_mapped = int(url in BLAT_GENOMES)
    return [
        assembly['assembly.default'],
        assembly['assembly.name'],
        assembly.get('assembly.accession', None),
        assembly.get('assembly.ucsc_alias', None),
        assembly.get('species.common_name', None),
        reconcile_taxids(assembly['species.taxonomy_id']),
        url,
        assembly['species.division'],
        domain_url(assembly['species.division']),
        example_location['chromosome'],
        example_location['start'],
        example_location['end'],
        is_mapped,
    ]


def write(handle, output):

    data = parse(handle, EXAMPLE_LOCATIONS)
    csv.writer(output).writerow(data)
{'fraction_non_alphanumeric': 0.12063953488372094, 'fraction_numerical': 0.04136851520572451, 'mean_word_length': 2.8308351177730193, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2362909', 'n_tokens_mistral': 3304, 'n_tokens_neox': 2854, 'n_words': 702}