<commit_before>from Bio import  SeqIO
import math
import sys
import os.path
import cPickle
import zlib
from SWSeqRecord import SWSeqRecord
from Bio.Seq import Seq
from atk import Window
from Indexer import Indexer

class QIndexer (Indexer):
    DNA = ['A', 'T', 'C', 'G']
    
    def __init__(self, settings, logger, stepFactor = 0.1, reads= [], qgram=1):
        Indexer.__init__(self, settings, logger, stepFactor, reads)
        self.qgram = qgram
        self.character_list = None
        self.generate_character_list(qgram)

    def generate_character_list(self, level=0):
        if level != 0:
            if self.character_list == None:
                self.character_list = [y for y in self.DNA]
            else:
                self.character_list = [y+x for x in self.DNA for y in self.character_list]
            self.generate_character_list(level-1)

    def count(self, seq, window, start_index, end_index):
        results = []
        if len(seq) > 0 and end_index - start_index > 0:
            n = seq.count("N", start_index, end_index)
            length = float(end_index - start_index - n)
            if length > 0 :
                results = [int(self.compositionScale*seq.count(x, start_index, end_index) / (length-self.qgram+1)) for x in self.character_list]
            else:
                results = [0] * len(self.character_list)
        else:
                results = [0] * len(self.character_list)
        r = [window]
        r.extend(results)
        return( tuple(r) )

    def createIndexAndStore(self, sequence, fileName, retainInMemory=True):
        self.createIndex(sequence, fileName, retainInMemory)

    def pickleName(self, fileName, length):
        return fileName + ".Q" + str(self.qgram) + "." + str(length) + ".index"


    def findIndices(self,seq, start = 0.0, step=False):
        """ finds the seeding locations for the mapping process.
        Structure of locations:
        (hit, window, distance), with hit: (location, reference seq id)
        Full structure:
        ((location, reference seq id), window, distance)
        :param seq: sequence used for comparison
        :param start: minimum distance. Use default unless you're stepping through distance values
        :param step: set this to True when you're stepping through distance values. Hence: start at 0 <= distance < 0.01, then 0.01 <= distance < 0.02, etc  
        """<commit_msg>Changed QIndexer from nonoverlapping to overlapping index<commit_after>from Bio import  SeqIO
import math
import sys
import os.path
import cPickle
import zlib
import re

from SWSeqRecord import SWSeqRecord
from Bio.Seq import Seq
from Indexer import Indexer



class QIndexer (Indexer):
    DNA = ['A', 'T', 'C', 'G']
    
    def __init__(self, settings, logger, stepFactor = 0.1, reads= [], qgram=1):
        Indexer.__init__(self, settings, logger, stepFactor, reads)
        self.qgram = qgram
        self.character_list = None
        self.generate_character_list(qgram)

    def generate_character_list(self, level=0):
        if level != 0:
            if self.character_list == None:
                self.character_list = [y for y in self.DNA]
            else:
                self.character_list = [y+x for x in self.DNA for y in self.character_list]
            self.generate_character_list(level-1)

    def count(self, seq, window, start_index, end_index):
        results = []
        if len(seq) > 0 and end_index - start_index > 0:
            n = seq.count("N", start_index, end_index)
            length = float(end_index - start_index - n)
            if length > 0 :
                #results = [int(self.compositionScale*seq.count(x, start_index, end_index) / (length-self.qgram+1)) for x in self.character_list]
                results = [int(self.compositionScale*len(re.findall(r"(?=" + x + ")", str(seq[start_index: end_index]))) / (length-self.qgram+1)) for x in self.character_list]
            else:
                results = [0] * len(self.character_list)
        else:
                results = [0] * len(self.character_list)
        r = [window]
        r.extend(results)
        return( tuple(r) )

    def createIndexAndStore(self, sequence, fileName, retainInMemory=True):
        self.createIndex(sequence, fileName, retainInMemory)

    def pickleName(self, fileName, length):
        return fileName + ".Q" + str(self.qgram) + "." + str(length) + ".index"


    def findIndices(self,seq, start = 0.0, step=False):
        """ finds the seeding locations for the mapping process.
        Structure of locations:
        (hit, window, distance), with hit: (location, reference seq id)
        Full structure:
        ((location, reference seq id), window, distance)
        :param seq: sequence used for comparison
        :param start: minimum distance. Use default unless you're stepping through distance values
        :param step: set this to True when you're stepping through distance values. Hence: start at 0 <= distance < 0.01, then 0.01 <= distance < 0.02, etc  
        """
{'fraction_non_alphanumeric': 0.1054726368159204, 'fraction_numerical': 0.009751243781094527, 'mean_word_length': 2.5519434628975266, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6507445', 'n_tokens_mistral': 1514, 'n_tokens_neox': 1451, 'n_words': 458}