<commit_before>#!/usr/bin/python

from camoco.Camoco import Camoco
from camoco.RefGen import RefGen
from camoco.Tools import log
from camoco.Locus import Locus


from collections import defaultdict
from pandas import DataFrame
from scipy.stats import hypergeom
import matplotlib.pylab as plt
import itertools
import pandas as pd
import re
import sys

class Term(object):
    '''
        A Term is a just group of loci that are related.
    '''
    def __init__(self, id, name='', type='', desc='', locus_list=None, **kwargs):
        self.id = id
        self.name = name
        self.type = type
        self.desc = desc
        self.locus_list = set()
        self.attr = {}
        if locus_list:
            self.locus_list = set(locus_list)
        for key, val in kwargs.items():
            self.attrs[key] = val

    def __len__(self):
        '''
            Returns the number of loci in the term.
        '''
        return len(self.locus_list)

    def add_locus(self, locus):
        '''
            Adds a locus to the Term.
        '''
        self.locus_list.add(locus)

    def flanking_snps(self, gene, window_size=100000):
        '''
            returns any nearby Term SNPs to a gene
        '''
        return [locus for locus in self.locus_list if abs(gene-locus) <= window_size]

    def effective_snps(self, window_size=None, max_genes_between=1):
        '''
            Collapse down loci that have overlapping windows.
            Also collapses down snps that have
        '''
        locus_list = sorted(self.locus_list)
        if window_size is not None:
            for locus in locus_list:
                locus.window = window_size
        collapsed = [locus_list.pop(0)]
        for locus in locus_list:
            # if they have overlapping windows, collapse
            if locus in collapsed[-1]:
                # Collapse if the windows overlap
                collapsed[-1] = collapsed[-1] + locus
            else:
                collapsed.append(locus)
        log('{}: Found {} SNPs -> {} effective SNPs', self.name, len(self.locus_list), len(collapsed))
        return collapsed<commit_msg>Fix another name/id bug.<commit_after>#!/usr/bin/python

from camoco.Camoco import Camoco
from camoco.RefGen import RefGen
from camoco.Tools import log
from camoco.Locus import Locus


from collections import defaultdict
from pandas import DataFrame
from scipy.stats import hypergeom
import matplotlib.pylab as plt
import itertools
import pandas as pd
import re
import sys

class Term(object):
    '''
        A Term is a just group of loci that are related.
    '''
    def __init__(self, id, name='', type='', desc='', locus_list=None, **kwargs):
        self.id = id
        self.name = name
        self.type = type
        self.desc = desc
        self.locus_list = set()
        self.attr = {}
        if locus_list:
            self.locus_list = set(locus_list)
        for key, val in kwargs.items():
            self.attrs[key] = val

    def __len__(self):
        '''
            Returns the number of loci in the term.
        '''
        return len(self.locus_list)

    def add_locus(self, locus):
        '''
            Adds a locus to the Term.
        '''
        self.locus_list.add(locus)

    def flanking_snps(self, gene, window_size=100000):
        '''
            returns any nearby Term SNPs to a gene
        '''
        return [locus for locus in self.locus_list if abs(gene-locus) <= window_size]

    def effective_snps(self, window_size=None, max_genes_between=1):
        '''
            Collapse down loci that have overlapping windows.
            Also collapses down snps that have
        '''
        locus_list = sorted(self.locus_list)
        if window_size is not None:
            for locus in locus_list:
                locus.window = window_size
        collapsed = [locus_list.pop(0)]
        for locus in locus_list:
            # if they have overlapping windows, collapse
            if locus in collapsed[-1]:
                # Collapse if the windows overlap
                collapsed[-1] = collapsed[-1] + locus
            else:
                collapsed.append(locus)
        log('{}: Found {} SNPs -> {} effective SNPs', self.id, len(self.locus_list), len(collapsed))
        return collapsed
{'fraction_non_alphanumeric': 0.09059070744805044, 'fraction_numerical': 0.005136586504786365, 'mean_word_length': 2.0426136363636362, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1502090', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1215, 'n_words': 392}