<commit_before>from libc.string cimport strsep, strdup, strcmp
from libc.stdlib cimport atoi, free

cimport cython

from pydigree.cydigree.datastructures import SparseArray

def vcf_allele_parser(datastr, int desired, int nallele):
    if not datastr:
        return None
    databytes = datastr.encode('utf8')
    cdef char* data = databytes

    outp = SparseArray(nallele, 0)
    cdef char* datadup = strdup(data)

    cdef char* delim = " \t"
    cdef char* subdelim = ':'
    cdef char* alleledelim = "/|" 
    
    cdef char* geno_tok
    cdef char* allele_tok

    cdef int allele = 0
    cdef int alleleidx = 0
    cdef int subtokidx = 0
    
    cdef char* token = strsep(&datadup, delim)
    while token:
        subtokidx = 0
        
        if token[0]:
            geno_tok = strsep(&token, subdelim)
            while subtokidx < desired:
                # Fast-forward to the genotype token
                geno_tok = strsep(&token, subdelim)
                subtokidx += 1


            if (strcmp(geno_tok, "./.") == 0) or (strcmp(geno_tok, ".|.") == 0):
                # Missing data
                outp.set_item(alleleidx, -1)
                outp.set_item(alleleidx + 1,-1)
                alleleidx += 2 

            elif (strcmp(geno_tok, "0/0") == 0) or (strcmp(geno_tok, "0|0") == 0):
                # Most common scenario
                alleleidx += 2

            else:
                # Allele 1
                allele_tok = strsep(&geno_tok, alleledelim)
                allele = atoi(allele_tok)
                
                if allele != 0:
                    outp.set_item(alleleidx, allele)

                alleleidx += 1

                # Allele 2
                allele_tok = strsep(&geno_tok, alleledelim)
                allele = atoi(allele_tok)

                if allele != 0:
                    outp.set_item(alleleidx, allele)

                alleleidx += 1
        
        token = strsep(&datadup, delim)
<commit_msg>some reorg<commit_after>from libc.string cimport strsep, strdup, strcmp, strlen
from libc.stdlib cimport atoi, free
from libc.stdint cimport int8_t

cimport cython

from pydigree.cydigree.datastructures import SparseArray

def vcf_allele_parser(datastr, int desired, int nallele):
    if not datastr:
        return None
    databytes = datastr.encode('utf8')
    cdef char* data = databytes

    outp = SparseArray(nallele, 0)
    cdef char* datadup = strdup(data)

    cdef char* delim = " \t"
    cdef char* subdelim = ':'
    cdef char* alleledelim = "/|" 
    
    cdef char* geno_tok
    cdef char* allele_tok

    cdef int allele = 0
    cdef int alleleidx = 0
    cdef int subtokidx = 0
    
    cdef char* token = strsep(&datadup, delim)
    while token:
        subtokidx = 0
        
        if token[0]:
            geno_tok = strsep(&token, subdelim)
            while subtokidx < desired:
                # Fast-forward to the genotype token
                geno_tok = strsep(&token, subdelim)
                subtokidx += 1

            if (strcmp(geno_tok, "0/0") == 0) or (strcmp(geno_tok, "0|0") == 0):
                # Most common scenario
                alleleidx += 2
            
            elif (strcmp(geno_tok, "./.") == 0) or (strcmp(geno_tok, ".|.") == 0):
                # Missing data
                outp.set_item(alleleidx, -1)
                outp.set_item(alleleidx + 1,-1)
                alleleidx += 2 

            elif strlen(geno_tok) == 3:
                allele = <int8_t>geno_tok[0] - 48 # '0' is ascii/utf8 48
                if allele != 0:
                    outp.set_item(alleleidx, allele)
                alleleidx += 1

                allele = <int8_t>geno_tok[2] - 48
                if allele != 0:
                    outp.set_item(alleleidx, allele)
                alleleidx += 1 

            else:
                # Allele 1
                allele_tok = strsep(&geno_tok, alleledelim)
                allele = atoi(allele_tok)
                
                if allele != 0:
                    outp.set_item(alleleidx, allele)

                alleleidx += 1

                # Allele 2
                allele_tok = strsep(&geno_tok, alleledelim)
                allele = atoi(allele_tok)

                if allele != 0:
                    outp.set_item(alleleidx, allele)

                alleleidx += 1
        
        token = strsep(&datadup, delim)

{'fraction_non_alphanumeric': 0.09353664087391898, 'fraction_numerical': 0.016385980883022302, 'mean_word_length': 1.4971590909090908, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6588510', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1374, 'n_words': 343}