<commit_before>import logging
import csv
import re

from dipper.sources.OMIMSource import OMIMSource
from dipper.models.Genotype import Genotype
from dipper.models.Model import Model
from dipper.models.GenomicFeature import Feature, makeChromID


LOG = logging.getLogger(__name__)


class HGNC(OMIMSource):
    """
    This is the processing module for HGNC.

    We create equivalences between HGNC identifiers and ENSEMBL and NCBIGene.
    We also add the links to cytogenic locations for the gene features.

    """

    EBIFTP = 'ftp://ftp.ebi.ac.uk/pub/databases/genenames/'
    files = {
        'genes': {
            'file': 'hgnc_complete_set.txt',
            'url': EBIFTP + 'new/tsv/hgnc_complete_set.txt',
            'columns': [
                'hgnc_id',
                'symbol',
                'name',
                'locus_group',
                'locus_type',
                'status',
                'location',
                'location_sortable',
                'alias_symbol',
                'alias_name',
                'prev_symbol',
                'prev_name',
                'gene_family',
                'gene_family_id',
                'date_approved_reserved',
                'date_symbol_changed',
                'date_name_changed',
                'date_modified',
                'entrez_id',
                'ensembl_gene_id',
                'vega_id',
                'ucsc_id',
                'ena',
                'refseq_accession',
                'ccds_id',
                'uniprot_ids',
                'pubmed_id',
                'mgd_id',
                'rgd_id',
                'lsdb',
                'cosmic',
                'omim_id',
                'mirbase',
                'homeodb',
                'snornabase',
                'bioparadigms_slc',
                'orphanet',
                'pseudogene.org',
                'horde_id',
                'merops',
                'imgt',
                'iuphar',
                'kznf_gene_catalog',
                'mamit-trnadb',
                'cd',
                'lncrnadb',
                'enzyme_id',
                'intermediate_filament_db',
                'rna_central_ids',
                'lncipedia',
                'gtrnadb'
            ],
        },
    }

    def __init__(self, graph_type, are_bnodes_skolemized,
                 tax_ids=None, gene_ids=None):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'hgnc',
            ingest_title='HGNC',
            ingest_url='https://www.genenames.org/',
            license_url=None,
            data_rights='ftp://ftp.ebi.ac.uk/pub/databases/genenames/README.txt',
            # file_handle=None
        )

        self.tax_ids = tax_ids
        self.gene_ids = gene_ids

        self.gene_ids = []

        if 'gene' not in self.all_test_ids:
            LOG.warning("not configured with gene test ids.")
        else:
            self.gene_ids = self.all_test_ids['gene']
        self.hs_txid = self.globaltt['Homo sapiens']

    def fetch(self, is_dl_forced=False):
        self.get_files(is_dl_forced)

    def parse(self, limit=None):
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)
        if self.test_only:
            self.test_mode = True
        LOG.info("Parsing files...")
        self._process_genes(limit)
        LOG.info("Done parsing files.")

    def _process_genes(self, limit=None):

        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph

        geno = Genotype(graph)
        model = Model(graph)

        raw = '/'.join((self.rawdir, self.files['genes']['file']))
        col = self.files['genes']['columns']
        LOG.info("Processing HGNC genes")

        chr_pattern = re.compile(r'(\d+|X|Y|Z|W|MT)[pq$]')
        band_pattern = re.compile(r'([pq][A-H\d]?\d?(?:\.\d+)?)')

        with open(raw, 'r', encoding="utf8") as csvfile:
            filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')

            row = next(filereader)
            if not self.check_fileheader(col, row):
                pass

            for row in filereader:
                # To generate:
                # head -1 hgnc_complete_set.txt.1 | tr '\t' '\n' |
                # sed "s/\(.*\)/\1 = row[col.index(\'\1\')]/g"

                hgnc_id = row[col.index('hgnc_id')].strip()
                symbol = row[col.index('symbol')].strip()
                name = row[col.index('name')].strip()
                # locus_group = row[col.index('locus_group')]
                locus_type = row[col.index('locus_type')].strip()
                # status = row[col.index('status')]
                location = row[col.index('location')].strip()
                # location_sortable = row[col.index('location_sortable')]
                # alias_symbol = row[col.index('alias_symbol')]
                # alias_name = row[col.index('alias_name')]
                # prev_symbol = row[col.index('prev_symbol')]
                # prev_name = row[col.index('prev_name')]
                # gene_family = row[col.index('gene_family')]
                # gene_family_id = row[col.index('gene_family_id')]
                # date_approved_reserved = row[col.index('date_approved_reserved')]
                # date_symbol_changed = row[col.index('date_symbol_changed')]
                # date_name_changed = row[col.index('date_name_changed')]
                # date_modified = row[col.index('date_modified')]
                entrez_id = row[col.index('entrez_id')].strip()
                ensembl_gene_id = row[col.index('ensembl_gene_id')].strip()
                # vega_id = row[col.index('vega_id')]
                # ucsc_id = row[col.index('ucsc_id')]
                # ena = row[col.index('ena')]
                # refseq_accession = row[col.index('refseq_accession')]
                # ccds_id = row[col.index('ccds_id')]
                # uniprot_ids = row[col.index('uniprot_ids')]
                pubmed_ids = row[col.index('pubmed_id')].strip()  # pipe seperated!
                # mgd_id = row[col.index('mgd_id')]
                # rgd_id = row[col.index('rgd_id')]
                # lsdb = row[col.index('lsdb')]
                # cosmic = row[col.index('cosmic')]
                omim_ids = row[col.index('omim_id')].strip()  # pipe seperated!
                # mirbase = row[col.index('mirbase')]
                # homeodb = row[col.index('homeodb')]
                # snornabase = row[col.index('snornabase')]<commit_msg>correct comment spelling<commit_after>import logging
import csv
import re

from dipper.sources.OMIMSource import OMIMSource
from dipper.models.Genotype import Genotype
from dipper.models.Model import Model
from dipper.models.GenomicFeature import Feature, makeChromID


LOG = logging.getLogger(__name__)


class HGNC(OMIMSource):
    """
    This is the processing module for HGNC.

    We create equivalences between HGNC identifiers and ENSEMBL and NCBIGene.
    We also add the links to cytogenic locations for the gene features.

    """

    EBIFTP = 'ftp://ftp.ebi.ac.uk/pub/databases/genenames/'
    files = {
        'genes': {
            'file': 'hgnc_complete_set.txt',
            'url': EBIFTP + 'new/tsv/hgnc_complete_set.txt',
            'columns': [
                'hgnc_id',
                'symbol',
                'name',
                'locus_group',
                'locus_type',
                'status',
                'location',
                'location_sortable',
                'alias_symbol',
                'alias_name',
                'prev_symbol',
                'prev_name',
                'gene_family',
                'gene_family_id',
                'date_approved_reserved',
                'date_symbol_changed',
                'date_name_changed',
                'date_modified',
                'entrez_id',
                'ensembl_gene_id',
                'vega_id',
                'ucsc_id',
                'ena',
                'refseq_accession',
                'ccds_id',
                'uniprot_ids',
                'pubmed_id',
                'mgd_id',
                'rgd_id',
                'lsdb',
                'cosmic',
                'omim_id',
                'mirbase',
                'homeodb',
                'snornabase',
                'bioparadigms_slc',
                'orphanet',
                'pseudogene.org',
                'horde_id',
                'merops',
                'imgt',
                'iuphar',
                'kznf_gene_catalog',
                'mamit-trnadb',
                'cd',
                'lncrnadb',
                'enzyme_id',
                'intermediate_filament_db',
                'rna_central_ids',
                'lncipedia',
                'gtrnadb'
            ],
        },
    }

    def __init__(self, graph_type, are_bnodes_skolemized,
                 tax_ids=None, gene_ids=None):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'hgnc',
            ingest_title='HGNC',
            ingest_url='https://www.genenames.org/',
            license_url=None,
            data_rights='ftp://ftp.ebi.ac.uk/pub/databases/genenames/README.txt',
            # file_handle=None
        )

        self.tax_ids = tax_ids
        self.gene_ids = gene_ids

        self.gene_ids = []

        if 'gene' not in self.all_test_ids:
            LOG.warning("not configured with gene test ids.")
        else:
            self.gene_ids = self.all_test_ids['gene']
        self.hs_txid = self.globaltt['Homo sapiens']

    def fetch(self, is_dl_forced=False):
        self.get_files(is_dl_forced)

    def parse(self, limit=None):
        if limit is not None:
            LOG.info("Only parsing first %d rows", limit)
        if self.test_only:
            self.test_mode = True
        LOG.info("Parsing files...")
        self._process_genes(limit)
        LOG.info("Done parsing files.")

    def _process_genes(self, limit=None):

        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph

        geno = Genotype(graph)
        model = Model(graph)

        raw = '/'.join((self.rawdir, self.files['genes']['file']))
        col = self.files['genes']['columns']
        LOG.info("Processing HGNC genes")

        chr_pattern = re.compile(r'(\d+|X|Y|Z|W|MT)[pq$]')
        band_pattern = re.compile(r'([pq][A-H\d]?\d?(?:\.\d+)?)')

        with open(raw, 'r', encoding="utf8") as csvfile:
            filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')

            row = next(filereader)
            if not self.check_fileheader(col, row):
                pass

            for row in filereader:
                # To generate:
                # head -1 hgnc_complete_set.txt.1 | tr '\t' '\n' |
                # sed "s/\(.*\)/\1 = row[col.index(\'\1\')]/g"

                hgnc_id = row[col.index('hgnc_id')].strip()
                symbol = row[col.index('symbol')].strip()
                name = row[col.index('name')].strip()
                # locus_group = row[col.index('locus_group')]
                locus_type = row[col.index('locus_type')].strip()
                # status = row[col.index('status')]
                location = row[col.index('location')].strip()
                # location_sortable = row[col.index('location_sortable')]
                # alias_symbol = row[col.index('alias_symbol')]
                # alias_name = row[col.index('alias_name')]
                # prev_symbol = row[col.index('prev_symbol')]
                # prev_name = row[col.index('prev_name')]
                # gene_family = row[col.index('gene_family')]
                # gene_family_id = row[col.index('gene_family_id')]
                # date_approved_reserved = row[col.index('date_approved_reserved')]
                # date_symbol_changed = row[col.index('date_symbol_changed')]
                # date_name_changed = row[col.index('date_name_changed')]
                # date_modified = row[col.index('date_modified')]
                entrez_id = row[col.index('entrez_id')].strip()
                ensembl_gene_id = row[col.index('ensembl_gene_id')].strip()
                # vega_id = row[col.index('vega_id')]
                # ucsc_id = row[col.index('ucsc_id')]
                # ena = row[col.index('ena')]
                # refseq_accession = row[col.index('refseq_accession')]
                # ccds_id = row[col.index('ccds_id')]
                # uniprot_ids = row[col.index('uniprot_ids')]
                pubmed_ids = row[col.index('pubmed_id')].strip()  # pipe separated!
                # mgd_id = row[col.index('mgd_id')]
                # rgd_id = row[col.index('rgd_id')]
                # lsdb = row[col.index('lsdb')]
                # cosmic = row[col.index('cosmic')]
                omim_ids = row[col.index('omim_id')].strip()  # pipe separated!
                # mirbase = row[col.index('mirbase')]
                # homeodb = row[col.index('homeodb')]
                # snornabase = row[col.index('snornabase')]
{'fraction_non_alphanumeric': 0.13064944929737943, 'fraction_numerical': 0.0007595898214963919, 'mean_word_length': 1.5689756097560976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1034598', 'n_tokens_mistral': 4303, 'n_tokens_neox': 3937, 'n_words': 629}