<commit_before>import itertools
from hashlib import md5


def get_tsv_header(tsvfn):
    with open(tsvfn) as fp:
        return next(fp).strip().split('\t')


def generate_tsv_lines_multifile(fns, header):
    return itertools.chain.from_iterable([generate_tsv_psms(fn, header)
                                          for fn in fns])


def generate_tsv_psms(fn, header):
    """Returns dicts with header-keys and psm statistic values"""
    with open(fn) as fp:
        next(fp)  # skip header
        for line in fp:
            yield {x: y for (x, y) in zip(header, line.strip().split('\t'))}


def get_header_index(header, index_options):
    """Module internal function.
    Loops index_options, for each option this function returns the
    index of the first found match in the header.
    If not found throws a ValueError"""
    for option in index_options:
        try:
            index = header.index(option)
        except ValueError:
            continue
        else:
            return index
    raise ValueError('Cannot find any of proposed header indices [ {0} ] in'
                     ' MzIdentML tsv header'.format(', '.join(index_options)))


def get_mzidtsv_lines_scannr_specfn(fn):
    """Returns generator of lines of tsv, skipping header, as split lists,
    and a tuple containing (spectra file, scan nr)."""
    with open(fn) as fp:
        header = next(fp).strip().split('\t')
        scan_ix = get_header_index(header, ['ScanNum', 'scannr',
                                            'scan_nr', 'Scan number'])
        fn_ix = get_header_index(header, ['#SpecFile', 'spectra_file',
                                          'specfile'])

        for line in fp:
            line = line.strip().split('\t')
            yield line, (line[fn_ix], line[scan_ix])


def get_peptide_proteins(line, specfn, scannr, unroll=False):
    """From a line, generate a peptide_id (MD5 of specfile and scannr).
    Returns that id with the peptide sequence and protein accessions.
    Return values:
        peptide_id      -   str
        peptideseq      -   str
        proteins        -   list of str
    """
    peptideseq = line[9]
    peptide_id = md5('{0}{1}'.format(specfn, scannr)
                     .encode('utf-8')).hexdigest()
    if unroll:
        peptideseq = peptideseq.split('.')[1]
        proteins = [line[10]]<commit_msg>Refactoring tsv reader for cleaner code<commit_after>import itertools
from hashlib import md5


def get_tsv_header(tsvfn):
    with open(tsvfn) as fp:
        return next(fp).strip().split('\t')


def generate_tsv_lines_multifile(fns, header):
    return itertools.chain.from_iterable([generate_tsv_psms(fn, header)
                                          for fn in fns])


def generate_tsv_psms(fn, header):
    """Returns dicts with header-keys and psm statistic values"""
    for line in generate_tsv_psms_line(fn):
        yield {x: y for (x, y) in zip(header, line.strip().split('\t'))}


def generate_tsv_psms_line(fn):
    with open(fn) as fp:
        next(fp)  # skip header
        for line in fp:
            yield line


def get_header_index(header, index_options):
    """Module internal function.
    Loops index_options, for each option this function returns the
    index of the first found match in the header.
    If not found throws a ValueError"""
    for option in index_options:
        try:
            index = header.index(option)
        except ValueError:
            continue
        else:
            return index
    raise ValueError('Cannot find any of proposed header indices [ {0} ] in'
                     ' MzIdentML tsv header'.format(', '.join(index_options)))


def get_mzidtsv_lines_scannr_specfn(fn):
    """Returns generator of lines of tsv, skipping header, as split lists,
    and a tuple containing (spectra file, scan nr)."""
    header = get_tsv_header(fn)
    scan_ix = get_header_index(header, ['ScanNum', 'scannr',
                                        'scan_nr', 'Scan number'])
    fn_ix = get_header_index(header, ['#SpecFile', 'spectra_file',
                                      'specfile'])
    for line in generate_tsv_psms_line(fn):
        line = line.strip().split('\t')
        yield line, (line[fn_ix], line[scan_ix])

        for line in fp:
            line = line.strip().split('\t')
            yield line, (line[fn_ix], line[scan_ix])


def get_peptide_proteins(line, specfn, scannr, unroll=False):
    """From a line, generate a peptide_id (MD5 of specfile and scannr).
    Returns that id with the peptide sequence and protein accessions.
    Return values:
        peptide_id      -   str
        peptideseq      -   str
        proteins        -   list of str
    """
    peptideseq = line[9]
    peptide_id = md5('{0}{1}'.format(specfn, scannr)
                     .encode('utf-8')).hexdigest()
    if unroll:
        peptideseq = peptideseq.split('.')[1]
        proteins = [line[10]]
{'fraction_non_alphanumeric': 0.1058727900833164, 'fraction_numerical': 0.004470636049583418, 'mean_word_length': 2.27478376580173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '186870', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1468, 'n_words': 436}