<commit_before>#Copyright 2012 <NAME>
#<EMAIL>
#http://jfi.uchicago.edu/~tcaswell
#
#This program is free software; you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation; either version 3 of the License, or (at
#your option) any later version.
#
#This program is distributed in the hope that it will be useful, but
#WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
#General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with this program; if not, see <http://www.gnu.org/licenses>.
from __future__ import division

import numpy as np
import scipy

class Hash_table(object):
    '''
    Basic hash table to fast look up of 
    '''
    class Out_of_hash_excpt(Exception):
        pass
    def __init__(self,dims,box_size):
        self.dims = dims                  # the dimensions of the data
        self.box_size = box_size          # the size of boxes to use in the units of the data
        self.hash_dims = np.ceil(np.array(dims)/box_size)

        self.hash_table = [[] for j in range(int(np.prod(self.hash_dims)))]
        self.spat_dims = len(dims)        # how many spatial dimensions
        self.cached_shifts = None
        self.cached_rrange = None
    def get_region(self,point, rrange):
        '''
        Returns all the particles with in the region of minimum radius
        rrange in data units
        '''
        
        center = np.floor(point.pos/self.box_size)

        rrange = int(np.ceil(rrange/ self.box_size))

        # check if we have already computed the shifts
        if rrange == self.cached_rrange and self.cached_shifts is not None:
            shifts = self.cached_shifts   # if we have, use them
        # Other wise, generate them
        else:
            if self.spat_dims == 2:
                shifts = [np.array([j,k]) 
                            for j in range(-rrange,rrange + 1) 
                            for k in range(-rrange,rrange + 1)]
            elif self.spat_dims ==3:
                shifts = [np.array([j,k,m])
                            for j in range(-rrange,rrange + 1)
                            for k in range(-rrange,rrange + 1)
                            for m in range(-rrange,rrange + 1)]
            else:
                raise NotImplementedError('only 2 and 3 dimensions implemented')
            self.cached_rrange = rrange   # and save them
            self.cached_shifts = shifts
        region = []
        for s in shifts:
            try:
                region.extend(self.hash_table[self.cord_to_indx(center + s)])
            except Hash_table.Out_of_hash_excpt:
                pass
        return region
    
    def add_point(self,point):
        self.hash_table[self.hash_fun(point.pos)].append(point)

    def hash_fun(self,cord):
        return self.cord_to_indx(np.floor(np.asarray(cord)/self.box_size))

    def cord_to_indx(self,cord):
        """Converts coordinate position to an index.  """
        assert len(cord) == len(self.hash_dims)
        hash_size = self.hash_dims
        cord = np.asarray(cord)
        if any(cord  >= hash_size) or any(cord < 0):
            raise Hash_table.Out_of_hash_excpt("cord out of range")
        indx = int(sum(cord * np.cumprod(np.concatenate(([1],hash_size[1:])))))

        return indx

    
class Track(object):<commit_msg>tweaked Hash_table to in-line some of the code to reduce function calls.<commit_after>#Copyright 2012 <NAME>
#<EMAIL>
#http://jfi.uchicago.edu/~tcaswell
#
#This program is free software; you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation; either version 3 of the License, or (at
#your option) any later version.
#
#This program is distributed in the hope that it will be useful, but
#WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
#General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with this program; if not, see <http://www.gnu.org/licenses>.
from __future__ import division

import numpy as np
import scipy

class Hash_table(object):
    '''
    Basic hash table to fast look up of 
    '''
    class Out_of_hash_excpt(Exception):
        pass
    def __init__(self,dims,box_size):
        self.dims = dims                  # the dimensions of the data
        self.box_size = box_size          # the size of boxes to use in the units of the data
        self.hash_dims = np.ceil(np.array(dims)/box_size)

        self.hash_table = [[] for j in range(int(np.prod(self.hash_dims)))]
        self.spat_dims = len(dims)        # how many spatial dimensions
        self.cached_shifts = None
        self.cached_rrange = None
        self.strides = np.cumprod(np.concatenate(([1],self.hash_dims[1:])))
    def get_region(self,point, rrange):
        '''
        Returns all the particles with in the region of minimum radius
        rrange in data units
        '''
        hash_size = self.hash_dims        
        center = np.floor(point.pos/self.box_size)
        if any(center  >= hash_size) or any(center < 0):
            raise Hash_table.Out_of_hash_excpt("cord out of range")

        rrange = int(np.ceil(rrange/ self.box_size))

        # check if we have already computed the shifts
        if rrange == self.cached_rrange and self.cached_shifts is not None:
            shifts = self.cached_shifts   # if we have, use them
        # Other wise, generate them
        else:
            if self.spat_dims == 2:
                shifts = [np.array([j,k]) 
                            for j in range(-rrange,rrange + 1) 
                            for k in range(-rrange,rrange + 1)]
            elif self.spat_dims ==3:
                shifts = [np.array([j,k,m])
                            for j in range(-rrange,rrange + 1)
                            for k in range(-rrange,rrange + 1)
                            for m in range(-rrange,rrange + 1)]
            else:
                raise NotImplementedError('only 2 and 3 dimensions implemented')
            self.cached_rrange = rrange   # and save them
            self.cached_shifts = shifts
        region = []

        for s in shifts:
            cord = center + s
            if any(cord  >= hash_size) or any(cord < 0):
                continue
            indx = int(sum(cord * self.strides))
            region.extend(self.hash_table[indx])
        return region
    
    def add_point(self,point):
        """Adds the `point` to the hash table.  Assumes that :py:attr:`point.pos` exists and
        is the correct type to be handed to :py:func:`hash_fun`

        :param point: object representing the feature to add to the hash table


        can raise :py:exc:`~Hash_table.Out_of_hash_excpt`
    
        """
        cord = np.floor(np.asarray(point.pos)/self.box_size)
        hash_size = self.hash_dims
        if any(cord  >= hash_size) or any(cord < 0):
            raise Hash_table.Out_of_hash_excpt("cord out of range")
        indx = int(sum(cord * self.strides))
        self.hash_table[indx].append(point)


    
class Track(object):
{'fraction_non_alphanumeric': 0.0894973436861463, 'fraction_numerical': 0.004903964037597058, 'mean_word_length': 2.1429794520547945, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7516047', 'n_tokens_mistral': 2176, 'n_tokens_neox': 2117, 'n_words': 694}