<commit_before>#! /usr/bin/env python3

'''
    Isolate subset of ClinVar XML for a TestSet based on Various IDs

'''
import os
import re
# import yaml
import gzip
import logging
import argparse
import xml.etree.ElementTree as ET


LOG = logging.getLogger(__name__)

# The name of the ingest we are doing
IPATH = re.split(r'/', os.path.realpath(__file__))
(INAME, DOTPY) = re.split(r'\.', IPATH[-1].lower())
RPATH = '/' + '/'.join(IPATH[1:-2])
files = {
    'f1': {
        'file': 'ClinVarFullRelease_00-latest.xml.gz',
        'url': 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_00-latest.xml.gz'}
}

# handle arguments for IO
ARGPARSER = argparse.ArgumentParser()

# INPUT
ARGPARSER.add_argument(
    '-f', '--filename', default=files['f1']['file'],
    help="input filename. default: '" + files['f1']['file'] + "'")

ARGPARSER.add_argument(
    '-i', '--inputdir', default=RPATH + '/raw/clinvarxml_alpha',
    help="input path. default: '" + RPATH + '/raw/clinvarxml_alpha' "'")

# OUTPUT
ARGPARSER.add_argument(
    '-d', "--destination", default=RPATH + '/raw/clinvarxml_alpha',
    help='output path. default: "' + RPATH + '/raw/clinvarxml_alpha')

ARGPARSER.add_argument(
    '-o', "--output", default=INAME + '.xml',
    help='file name to write to')

ARGPARSER.add_argument(
    '-t', "--testfile", default=RPATH + '/raw/test/testid.txt',
    help='file of IDs to capture')

ARGS = ARGPARSER.parse_args()

FILENAME = ARGS.inputdir + '/' + ARGS.filename

OUTPUT = ARGS.destination + '/' + ARGS.output

VARIANT = []
DISEASE = []
GENE = []
RCV = []

with open(ARGS.testfile) as f:
    for line in f:
        line = line.partition('#')[0].strip()  # no comment
        if line != "":
            (prfx, lcl_id) = re.split(r':', line, 2)
            #
            if prfx == 'ClinVar':
                RCV.append(lcl_id.strip())

            elif prfx == 'ClinVarVariant':
                VARIANT.append(lcl_id.strip())

            elif prfx == 'NCBIGene':
                GENE.append(lcl_id.strip())

            elif prfx == 'OMIM':
                DISEASE.append(lcl_id.strip())

LOG.warning('RCV has ' + str(len(RCV)))
LOG.warning('GENE has ' + str(len(GENE)))
LOG.warning('DISEASE has ' + str(len(DISEASE)))
LOG.warning('VARIANT has ' + str(len(VARIANT)))

#######################################################
# main loop over xml
# taken in chunks composed of ClinVarSet stanzas

with gzip.open(FILENAME, 'rt') as fh:
    tree = ET.iterparse(fh, events=('start', 'end'))
    # event, root = iter(tree).next()
    for event, element in tree:
        if event == 'start' and element.tag == 'ReleaseSet':
            ReleaseSet = element
            continue
        elif event == 'end' and element.tag == 'ClinVarSet':
            ClinVarSet = element
        else:
            continue

        # Clinvar sets to keep based on the inclusion of some identifier
        # it could be a ClinVarVariant ID or a RCV or an OMIM or a NCBIGene
        keep = False

        # /ReleaseSet/ClinVarSet/ReferenceClinVarAssertion/MeasureSet/@ID
        RCVAssertion = ClinVarSet.find('./ReferenceClinVarAssertion')

        # rcv_variants are not unique to a single ClinVarSet
        # rcv_variant = int(RCVAssertion.find('./MeasureSet').get('ID'))

        rcv_acc = RCVAssertion.find('./ClinVarAccession').get('Acc')

        # Disease
        for RCV_TraitSet in RCVAssertion.findall('TraitSet'):
            for RCV_Trait in RCV_TraitSet.findall('Trait[@Type="Disease"]'):
                for RCV_TraitXRef in RCV_Trait.findall('XRef[@DB="OMIM"]'):
                    rcv_disease_id = RCV_TraitXRef.get('ID')
                    if not keep and rcv_disease_id is not None \
                            and rcv_disease_id in DISEASE:
                        LOG.warning(rcv_disease_id + ' in DISEASE')
                        keep = True
        # Gene
        # /RCV/MeasureSet/Measure/MeasureRelationship[@Type]/XRef[@DB="Gene"]/@ID
        RCV_MeasureSet = RCVAssertion.find('./MeasureSet')
        if RCV_MeasureSet is None:
            #GenotypeSet
            continue
        else:
            for rms in RCV_MeasureSet:
                RCV_Measure = rms.findall('./Measure')
                RCV_MRel = rms.find('./MeasureRelationship')
                if RCV_MRel is not None and not keep:
                    RCV_Gene = RCV_MRel.find('./XRef[@DB="Gene"]')
                    if RCV_Gene is not None:
                        rcv_ncbigene_id = RCV_Gene.get('ID')
                        if rcv_ncbigene_id is not None \
                                and rcv_ncbigene_id in GENE:
                            LOG.warning(rcv_ncbigene_id + ' in GENE')
                            keep = True<commit_msg>update path/names; comments<commit_after>#! /usr/bin/env python3

'''
    Isolate subset of ClinVar XML for a TestSet based on Various IDs

'''
import os
import re
# import yaml
import gzip
import logging
import argparse
import xml.etree.ElementTree as ET


LOG = logging.getLogger(__name__)

# The name of the ingest we are doing
IPATH = re.split(r'/', os.path.realpath(__file__))
(INAME, DOTPY) = re.split(r'\.', IPATH[-1].lower())
RPATH = '/' + '/'.join(IPATH[1:-2])
FTP = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml'
files = {
    'f1': {
        'file': 'ClinVarFullRelease_00-latest.xml.gz',
        'url': '/'.join((FTP, 'ClinVarFullRelease_00-latest.xml.gz'))}
}

# handle arguments for IO
ARGPARSER = argparse.ArgumentParser()

# INPUT
ARGPARSER.add_argument(
    '-f', '--filename', default=files['f1']['file'],
    help="input filename. default: '" + files['f1']['file'] + "'")

ARGPARSER.add_argument(
    '-i', '--inputdir', default=RPATH + '/raw/clinvar',
    help="input path. default: '" + RPATH + '/raw/clinvar' "'")

# OUTPUT
ARGPARSER.add_argument(
    '-d', "--destination", default=RPATH + '/raw/clinvar',
    help='output path. default: "' + RPATH + '/raw/clinvar')

ARGPARSER.add_argument(
    '-o', "--output", default=INAME + '.xml',
    help='file name to write to')

ARGPARSER.add_argument(
    '-t', "--testfile", default=RPATH + '/raw/test/clinvar_curie.txt',
    help='file w/list of CURIEs to capture')

ARGS = ARGPARSER.parse_args()

FILENAME = ARGS.inputdir + '/' + ARGS.filename

OUTPUT = ARGS.destination + '/' + ARGS.output

VARIANT = []    # VCV
DISEASE = []    # OMIM ...
GENE = []       # NCBIGene  ...
RCV = []        #

with open(ARGS.testfile) as f:
    for line in f:
        line = line.partition('#')[0].strip()  # no comment
        if line != "":
            (prfx, lcl_id) = re.split(r':', line, 2)
            #
            if prfx in ['RCV', 'ClinVar']:
                RCV.append(lcl_id.strip())

            elif prfx == 'ClinVarVariant':
                VARIANT.append(lcl_id.strip())

            elif prfx == 'NCBIGene':
                GENE.append(lcl_id.strip())

            elif prfx == 'OMIM':
                DISEASE.append(lcl_id.strip())

LOG.warning('RCV has ' + str(len(RCV)))
LOG.warning('GENE has ' + str(len(GENE)))
LOG.warning('DISEASE has ' + str(len(DISEASE)))
LOG.warning('VARIANT has ' + str(len(VARIANT)))

#######################################################
# main loop over xml
# taken in chunks composed of ClinVarSet stanzas

with gzip.open(FILENAME, 'rt') as fh:
    tree = ET.iterparse(fh, events=('start', 'end'))
    # event, root = iter(tree).next()
    for event, element in tree:
        if event == 'start' and element.tag == 'ReleaseSet':
            ReleaseSet = element
            continue
        elif event == 'end' and element.tag == 'ClinVarSet':
            ClinVarSet = element
        else:
            continue

        # Clinvar sets to keep based on the inclusion of some identifier
        # it could be a ClinVarVariant ID or a RCV or an OMIM or a NCBIGene
        keep = False

        # /ReleaseSet/ClinVarSet/ReferenceClinVarAssertion/MeasureSet/@ID
        RCVAssertion = ClinVarSet.find('./ReferenceClinVarAssertion')

        # rcv_variants are not unique to a single ClinVarSet
        # rcv_variant = int(RCVAssertion.find('./MeasureSet').get('ID'))

        rcv_acc = RCVAssertion.find('./ClinVarAccession').get('Acc')

        # Disease
        for RCV_TraitSet in RCVAssertion.findall('TraitSet'):
            for RCV_Trait in RCV_TraitSet.findall('Trait[@Type="Disease"]'):
                for RCV_TraitXRef in RCV_Trait.findall('XRef[@DB="OMIM"]'):
                    rcv_disease_id = RCV_TraitXRef.get('ID')
                    if not keep and rcv_disease_id is not None \
                            and rcv_disease_id in DISEASE:
                        LOG.warning(rcv_disease_id + ' in DISEASE')
                        keep = True
        # Gene
        # /RCV/MeasureSet/Measure/MeasureRelationship[@Type]/XRef[@DB="Gene"]/@ID
        RCV_MeasureSet = RCVAssertion.find('./MeasureSet')
        if RCV_MeasureSet is None:
            # GenotypeSet
            continue
        else:
            for rms in RCV_MeasureSet:
                RCV_Measure = rms.findall('./Measure')
                RCV_MRel = rms.find('./MeasureRelationship')
                if RCV_MRel is not None and not keep:
                    RCV_Gene = RCV_MRel.find('./XRef[@DB="Gene"]')
                    if RCV_Gene is not None:
                        rcv_ncbigene_id = RCV_Gene.get('ID')
                        if rcv_ncbigene_id is not None \
                                and rcv_ncbigene_id in GENE:
                            LOG.warning(rcv_ncbigene_id + ' in GENE')
                            keep = True
{'fraction_non_alphanumeric': 0.1481365159128978, 'fraction_numerical': 0.0027219430485762143, 'mean_word_length': 2.2471108089734875, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7459599', 'n_tokens_mistral': 3205, 'n_tokens_neox': 3092, 'n_words': 696}