<commit_before>from Bio import SeqIO
from Bio.Seq import Seq
from copy import copy 
import itertools
from collections import Counter
import logging
import datetime
import math 

def unique(seq):
    seen = set()
    seen_add = seen.add
    return [ x for x in seq if x not in seen and not seen_add(x)]


class Variant(object):

    def __init__(self, ref, pos, alt):
        self.ref = ref 
        self.pos = pos
        self.alt = alt

    def __str__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __repr__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __gt__(self, other):
        return self.pos > other.pos

    def __lt__(self, other):
        return self.pos < other.pos

    def overlapping(self, other):
        return self.pos == other.pos
        # return other.pos in self.ref_range 

    @property
    def ref_range(self):
        return range(self.pos, self.pos + len(self.ref))        

    @property
    def length(self):
        return abs(len(self.ref) - len(self.alt))

    @property
    def is_indel(self):
        """ Return whether or not the variant is an INDEL """
        if len(self.ref) > 1:
            return True
        if self.alt is None:
            return True
        elif len(self.alt) != len(self.ref):
            return True
        return False

    @property
    def is_deletion(self):
        """ Return whether or not the INDEL is a deletion """
        if self.is_indel:
            # just one alt allele
            if self.alt is None:
                return True
            if len(self.ref) > len(self.alt):
                return True<commit_msg>Refactored recursive context creator for clarity and to allow for overlapping indels<commit_after>from Bio import SeqIO
from Bio.Seq import Seq
from copy import copy 
import itertools
from collections import Counter
import logging
import datetime
import math 

def unique(seq):
    seen = set()
    seen_add = seen.add
    return [ x for x in seq if x not in seen and not seen_add(x)]


class Variant(object):

    def __init__(self, ref, pos, alt):
        self.ref = ref 
        self.pos = pos
        self.alt = alt

    def __str__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __repr__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __gt__(self, other):
        return self.pos > other.pos

    def __lt__(self, other):
        return self.pos < other.pos

    def overlapping(self, other):
        # return self.pos == other.pos
        return other.pos in self.ref_range 

    @property
    def ref_range(self):
        return range(self.pos, self.pos + len(self.ref))        

    @property
    def length(self):
        return abs(len(self.ref) - len(self.alt))

    @property
    def is_indel(self):
        """ Return whether or not the variant is an INDEL """
        if len(self.ref) > 1:
            return True
        if self.alt is None:
            return True
        elif len(self.alt) != len(self.ref):
            return True
        return False

    @property
    def is_deletion(self):
        """ Return whether or not the INDEL is a deletion """
        if self.is_indel:
            # just one alt allele
            if self.alt is None:
                return True
            if len(self.ref) > len(self.alt):
                return True
{'fraction_non_alphanumeric': 0.09754655631096659, 'fraction_numerical': 0.0005911912503694946, 'mean_word_length': 2.0, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5913844', 'n_tokens_mistral': 1035, 'n_tokens_neox': 986, 'n_words': 336}