<commit_before>import sys
import os
import re

class DynLoad(object):
    """Dynamically loads all .py([oc]?) files in a folder
    
    TODO: Build the exception classes
    
    """
    def __init__(self, path = None):
        if path:
            self.set_path(path)
        self.dyn_modules = {'loaded':set(), 'failed':set()}
        
    def set_path(self, directory):
        """Set the folder path to look for python files"""
        #======================================
        # verify the path given is a directory
        # if it is set the private path variable
        # accordingly
        #======================================
        if os.path.isdir(directory):
            sys.path.append(directory)
            self.__path = directory
        else:
            raise Exception('Invalid Path')
    
    def __find_python_files(self, path):
        """private interator function that searchs for python files"""
        #======================================
        # find files with non-numeric names that
        # end with .py .pyc .pyo or .pyd
        #======================================
        rx_find_py = re.compile('^(\w+)\.(?:py|pyc|pyo|pyd)$')
        
        #======================================
        # check each file found in the directory
        # to see if it is python, if it is
        # return the file to the calling function
        #======================================
        for f in os.listdir(self.__path):
            f_path = os.path.join(self.__path, f)
            if os.path.isfile(f_path):
                rx_result = rx_find_py.search(f)
                if rx_result:
                    yield rx_result.group(1)
    
    def load_modules(self):
        """import modules in the directory 
        those that cannot be loaded are skipped
        
        """
        for module in self.__find_python_files(self.__path):
            # Exclude anything we've tried to import
            # Or init.
            if module == '__init__' or \
               module in self.dyn_modules['failed'] or \
               module in self.dyn_modules['loaded'] :
                continue
            else:
                #======================================
                # try to load each variable into the
                # buildin python import function
                # if it fails, just continue
                #======================================
                try:
                    globals()['%s' % module] = __import__('%s' % module)
                except Exception as e:
                    print 'Error on startup:', e
                    self.dyn_modules['failed'].add(module)
                else:
                    self.dyn_modules['loaded'].add(module)
   
class AddRoute(object):
    """
    Decorator that tracks and allows generation of new routes within
    the python file itself rather that declared upfront.
    """
    _routes = []
    def __init__(self, arg):
        """ 
        route keeper arguments
        """
        self._arg = arg
        
    def __call__(self, route_handler):
        '''
        The part that calls the internal function
        '''
        # Add the route to the internal structure
        self._routes.append((self._arg, route_handler))
        def wrapper(*args, **kwargs):
            return route_handler(*args, **kwargs)
        return wrapper
    
    @classmethod   
    def get_routes(self):
        return self._routes
    
    @classmethod   
    def get_paths(self):
        for path in [l[0] for l in self._routes]:
            yield path<commit_msg>Update routestar.py<commit_after>import sys
import os
import re


class DynLoad(object):
    """Dynamically loads all .py([oc]?) files in a folder
    
    TODO: Build the exception classes
    
    """
    def __init__(self, path=None):
        if path:
            self.set_path(path)
        self.dyn_modules = {'loaded': set(), 'failed': set()}
        #======================================
        # find files with non-numeric names that
        # end with .py .pyc .pyo or .pyd
        #======================================
        self.rx_find_py = re.compile('^(\w+)\.(?:py|pyc|pyo|pyd)$')
        
    def set_path(self, directory):
        """Set the folder path to look for python files"""
        #======================================
        # verify the path given is a directory
        # if it is set the private path variable
        # accordingly
        #======================================
        if os.path.isdir(directory):
            sys.path.append(directory)
            self.__path = directory
        else:
            raise Exception('Invalid Path')
    
    def __find_python_files(self, path):
        """private interator function that searchs for python files"""
        #======================================
        # check each file found in the directory
        # to see if it is python, if it is
        # return the file to the calling function
        #======================================
        for f in os.listdir(self.__path):
            f_path = os.path.join(self.__path, f)
            if os.path.isfile(f_path):
                rx_result = self.rx_find_py.search(f)
                if rx_result:
                    yield rx_result.group(1)
    
    def load_modules(self):
        """import modules in the directory 
        those that cannot be loaded are skipped
        
        TODO: return imported and failed modules in a dict
        
        """
        for module in self.__find_python_files(self.__path):
            # Exclude anything we've tried to import
            # Or init.
            if module == '__init__' or \
               module in self.dyn_modules['failed'] or \
               module in self.dyn_modules['loaded'] :
                continue
            else:
                #======================================
                # try to load each variable into the
                # buildin python import function
                # if it fails, just continue
                #======================================
                try:
                    globals()['%s' % module] = __import__('%s' % module)
                except Exception as e:
                    print 'Error on startup:', e
                    self.dyn_modules['failed'].add(module)
                else:
                    self.dyn_modules['loaded'].add(module)


class AddRoute(object):
    """
    Decorator that tracks and allows generation of new routes within
    the python file itself rather that declared upfront.
    """
    _routes = []

    def __init__(self, arg, **kwargs):
        """ 
        route keeper arguments
        """
        self._arg = arg
        self._kwargs = kwargs
        
    def __call__(self, route_handler):
        """
        The part that calls the internal function
        """
        # Add the route to the internal structure
        if self._kwargs.get('r_type', None) == '404':
            self._routes.append((self._arg, route_handler))
        else:
            self._routes.insert(0, (self._arg, route_handler))

        def wrapper(*args, **kwargs):
            return route_handler(*args, **kwargs)
        return wrapper
    
    @classmethod   
    def get_routes(cls):
        return cls._routes
    
    @classmethod   
    def get_paths(cls):
        for path in [l[0] for l in cls._routes]:
            yield path
{'fraction_non_alphanumeric': 0.16592652402099314, 'fraction_numerical': 0.0010765711209796797, 'mean_word_length': 1.8130204390613172, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182183', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1833, 'n_words': 579}