<commit_before>import re

def find_all(a_str, sub):
    '''
    Generator to find the coordinates of the start of all
    occurrences of a substring
    '''
    start = 0
    while True:
        start = a_str.find(sub, start)
        if start == -1: return
        yield start
        start += len(sub) # use start += 1 to find overlapping matches

def parse_vcf(vcf_file_stream, vcf_lines, vcf_headers, add_sname=True, include_ref=False):
    header = ''
    samples = ''

    for l in vcf_file_stream:
        if l[0] == '#':
            if l[1] != '#':
                samples = l.rstrip().split('\t')[9:]
            else:
                # ignore fileDate
                if l[:10] == '##fileDate':
                    continue
                if l not in vcf_headers:
                    vcf_headers.append(l)
        else:
            A = l.split('\t')
            if not include_ref and ('GT' in A[8]):
                has_nonref = False
                for sample_field in A[9:]:
                    if not (sample_field.startswith('0/0') or sample_field.startswith('./.')):
                        has_nonref = True
                        break
                if not has_nonref:
                    continue

            if not 'SECONDARY' in A[7]:

                if add_sname and (samples != []):
                    A[7] += ';' + 'SNAME=' + ','.join(samples)
                    l = '\t'.join(A)


                if 'SVTYPE=BND' in A[7]:
                    m = re.search(r"(\[|\])(.*)(\[|\])",A[4])
                    o_chr,o_pos = m.group(2).split(':')

                    if (o_chr == A[0]) and (('--:' in A[7]) != ('++' in A[7])):
                        neg_s = A[7].find('--:')<commit_msg>Handle 8-column VCF for lsort<commit_after>import re

def find_all(a_str, sub):
    '''
    Generator to find the coordinates of the start of all
    occurrences of a substring
    '''
    start = 0
    while True:
        start = a_str.find(sub, start)
        if start == -1: return
        yield start
        start += len(sub) # use start += 1 to find overlapping matches

def parse_vcf(vcf_file_stream, vcf_lines, vcf_headers, add_sname=True, include_ref=False):
    header = ''
    samples = ''

    for l in vcf_file_stream:
        if l[0] == '#':
            if l[1] != '#':
                samples = l.rstrip().split('\t')[9:]
            else:
                # ignore fileDate
                if l[:10] == '##fileDate':
                    continue
                if l not in vcf_headers:
                    vcf_headers.append(l)
        else:
            A = l.split('\t')
            if not include_ref and (len(A) > 8 and 'GT' in A[8]):
                has_nonref = False
                for sample_field in A[9:]:
                    if not (sample_field.startswith('0/0') or sample_field.startswith('./.')):
                        has_nonref = True
                        break
                if not has_nonref:
                    continue

            if not 'SECONDARY' in A[7]:

                if add_sname and (samples != []):
                    A[7] += ';' + 'SNAME=' + ','.join(samples)
                    l = '\t'.join(A)


                if 'SVTYPE=BND' in A[7]:
                    m = re.search(r"(\[|\])(.*)(\[|\])",A[4])
                    o_chr,o_pos = m.group(2).split(':')

                    if (o_chr == A[0]) and (('--:' in A[7]) != ('++' in A[7])):
                        neg_s = A[7].find('--:')
{'fraction_non_alphanumeric': 0.1462850534836658, 'fraction_numerical': 0.012720439433362244, 'mean_word_length': 1.3066666666666666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4177575', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1052, 'n_words': 270}