<commit_before>
cdef extern from "stdio.h":
  ctypedef struct FILE:
    pass
  FILE *fopen(char *,char *)
  int fclose(FILE *)
  int sscanf(char *str,char *fmt,...)
  int sprintf(char *str,char *fmt,...)
  char *fgets(char *str,int size,FILE *ifile)

cdef extern from "ctype.h":
    int isspace(int)
    int isprint(int)

cdef extern from "string.h":
    char *strcpy(char *,char *)



def read_fasta_lengths(d,filename):
    cdef int i,seqLength
    cdef char tmp[32768],fastastart[4],*p
    cdef FILE *ifile

    ifile=fopen(filename,'r')
    if ifile==NULL:
        raise IOError('unable to open %s' % filename)
    id=None
    seqLength=0
    strcpy(fastastart,'>')
    p=fgets(tmp,32767,ifile) # READ THE FIRST LINE OF THE MAF FILE
    while p:
        if fastastart[0]==p[0]: #NEW SEQUENCE
            if id is not None and seqLength>0:
                d[id]=seqLength # SAVE THIS SEQ LENGTH
            id=str(p+1).split()[0]
            seqLength=0
        else:
            i=0
            while p[i]:
                if isprint(p[i]) and not isspace(p[i]):
                    seqLength=seqLength+1
                i=i+1
        p=fgets(tmp,32767,ifile) # READ THE FIRST LINE OF THE MAF FILE
    if id is not None and seqLength>0:
        d[id]=seqLength # SAVE THIS SEQ LENGTH
    fclose(ifile)
    <commit_msg>added pureseq file saving to enable fast sequence lookup<commit_after>
cdef extern from "stdio.h":
  ctypedef struct FILE:
    pass
  FILE *fopen(char *,char *)
  int fclose(FILE *)
  int sscanf(char *str,char *fmt,...)
  int sprintf(char *str,char *fmt,...)
  char *fgets(char *str,int size,FILE *ifile)
  int fputc(int,FILE *)

cdef extern from "ctype.h":
    int isspace(int)
    int isprint(int)

cdef extern from "string.h":
    char *strcpy(char *,char *)



def read_fasta_lengths(d,filename):
    cdef int i
    cdef long long seqLength,ipos,offset # MUST USE 64-BIT INT!!!
    cdef char tmp[32768],fastastart[4],*p
    cdef FILE *ifile,*ifile2

    ifile=fopen(filename,'r')
    if ifile==NULL:
        raise IOError('unable to open %s' % filename)
    outfile=filename+'.pureseq'
    ifile2=fopen(outfile,'w')
    if ifile2==NULL:
        raise IOError('unable to create %s' % (filename+'.pureseq'))
    id=None
    ipos=0
    seqLength=0
    strcpy(fastastart,'>')
    p=fgets(tmp,32767,ifile) # READ THE FIRST LINE OF THE MAF FILE
    while p:
        if fastastart[0]==p[0]: #NEW SEQUENCE
            if id is not None and seqLength>0:
                d[id]=seqLength,offset # SAVE THIS SEQ LENGTH
            id=str(p+1).split()[0]
            offset=ipos
            seqLength=0
        else:
            i=0
            while p[i]:
                if isprint(p[i]) and not isspace(p[i]):
                    seqLength=seqLength+1
                    fputc(p[i],ifile2)
                    ipos=ipos+1
                i=i+1
        p=fgets(tmp,32767,ifile) # READ THE FIRST LINE OF THE MAF FILE
    if id is not None and seqLength>0:
        d[id]=seqLength,offset # SAVE THIS SEQ LENGTH
    fclose(ifile)
    fclose(ifile2)
    
{'fraction_non_alphanumeric': 0.12035225048923678, 'fraction_numerical': 0.02054794520547945, 'mean_word_length': 2.319264069264069, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7271198', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1085, 'n_words': 287}