<commit_before>"""
    File handler to support different file extensions.
    Uses reflectometry's registry utility.
    
    The default readers are found in the 'readers' sub-module
    and registered by default at initialization time.
    
    To add a new default reader, one must register it in
    the register_readers method found in readers/__init__.py. 
    
    A utility method (find_plugins) is available to inspect 
    a directory (for instance, a user plug-in directory) and
    look for new readers/writers.
"""

"""
This software was developed by the University of Tennessee as part of the
Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
project funded by the US National Science Foundation. 

See the license text in license.txt

copyright 2008, University of Tennessee
"""

from data_util.registry import ExtensionRegistry
import os 
import sys
import logging
import time
from zipfile import ZipFile

# Default readers are defined in the readers sub-module
import readers

class Registry(ExtensionRegistry):
    """
        Registry class for file format extensions.
        Readers and writers are supported.
    """
    
    def __init__(self):
        super(Registry, self).__init__()
        
        ## Writers
        self.writers = {}
        
        ## List of wildcards
        self.wildcards = ['All (*.*)|*.*']
        
        ## Creation time, for testing
        self._created = time.time()
        
        # Register default readers
        readers.register_readers(self._identify_plugin)
        
    def find_plugins(self, dir):
        """
            Find readers in a given directory. This method
            can be used to inspect user plug-in directories to
            find new readers/writers.
            
            @param dir: directory to search into
            @return: number of readers found
        """
        readers_found = 0
        
        # Check whether the directory exists
        if not os.path.isdir(dir): 
            logging.warning("DataLoader couldn't load from %s" % dir)
            return readers_found
        
        for item in os.listdir(dir):
            full_path = os.path.join(dir, item)
            if os.path.isfile(full_path):
                
                # Process python files
                if item.endswith('.py'):
                    toks = os.path.splitext(os.path.basename(item))
                    try:
                        sys.path.insert(0, os.path.abspath(dir))
                        module = __import__(toks[0], globals(), locals())
                        if self._identify_plugin(module):
                            readers_found += 1
                    except :
                        logging.error("Loader: Error importing %s\n  %s" % (item, sys.exc_value))
                            
                # Process zip files
                elif item.endswith('.zip'):
                    try:
                        # Find the modules in the zip file
                        zfile = ZipFile(item)
                        nlist = zfile.namelist()
                        
                        sys.path.insert(0, item)
                        for mfile in nlist:
                            try:
                                # Change OS path to python path
                                fullname = mfile.replace('/', '.')
                                fullname = os.path.splitext(fullname)[0]
                                module = __import__(fullname, globals(), locals(), [""])
                                if self._identify_plugin(module):
                                    readers_found += 1
                            except:
                                logging.error("Loader: Error importing %s\n  %s" % (mfile, sys.exc_value))
                            
                    except:
                        logging.error("Loader: Error importing %s\n  %s" % (item, sys.exc_value))
                     
        return readers_found 
    
    def _identify_plugin(self, module):
        """
            Look into a module to find whether it contains a 
            Reader class. If so, add it to readers and (potentially)
            to the list of writers.
            @param module: module object
        """
        reader_found = False
        
        if hasattr(module, "Reader"):
            try:
                # Find supported extensions
                loader = module.Reader()
                for ext in loader.ext:
                    if ext not in self.loaders:
                        self.loaders[ext] = []
                    self.loaders[ext].insert(0,loader.read)
                    reader_found = True
                        
                    # Keep track of wildcards
                    for wcard in loader.type:
                        if wcard not in self.wildcards:
                            self.wildcards.append(wcard)
                            
                # Check whether writing is supported
                if hasattr(loader, 'write'):
                    for ext in loader.ext:
                        if ext not in self.writers:
                            self.writers[ext] = []
                        self.writers[ext].insert(0,loader.write)
            except:
                logging.error("Loader: Error accessing Reader in %s\n  %s" % (name, sys.exc_value))
        return reader_found

    def lookup_writers(self, path):
        """
        Return the loader associated with the file type of path.
        
        Raises ValueError if file type is not known.
        """        <commit_msg>ACS reads abs files when errored  from abs reader<commit_after>"""
    File handler to support different file extensions.
    Uses reflectometry's registry utility.
    
    The default readers are found in the 'readers' sub-module
    and registered by default at initialization time.
    
    To add a new default reader, one must register it in
    the register_readers method found in readers/__init__.py. 
    
    A utility method (find_plugins) is available to inspect 
    a directory (for instance, a user plug-in directory) and
    look for new readers/writers.
"""

"""
This software was developed by the University of Tennessee as part of the
Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
project funded by the US National Science Foundation. 

See the license text in license.txt

copyright 2008, University of Tennessee
"""

from data_util.registry import ExtensionRegistry
import os 
import sys
import logging
import time
from zipfile import ZipFile

# Default readers are defined in the readers sub-module
import readers

class Registry(ExtensionRegistry):
    """
        Registry class for file format extensions.
        Readers and writers are supported.
    """
    
    def __init__(self):
        super(Registry, self).__init__()
        
        ## Writers
        self.writers = {}
        
        ## List of wildcards
        self.wildcards = ['All (*.*)|*.*']
        
        ## Creation time, for testing
        self._created = time.time()
        
        # Register default readers
        readers.register_readers(self._identify_plugin)
        
    def find_plugins(self, dir):
        """
            Find readers in a given directory. This method
            can be used to inspect user plug-in directories to
            find new readers/writers.
            
            @param dir: directory to search into
            @return: number of readers found
        """
        readers_found = 0
        
        # Check whether the directory exists
        if not os.path.isdir(dir): 
            logging.warning("DataLoader couldn't load from %s" % dir)
            return readers_found
        
        for item in os.listdir(dir):
            full_path = os.path.join(dir, item)
            if os.path.isfile(full_path):
                
                # Process python files
                if item.endswith('.py'):
                    toks = os.path.splitext(os.path.basename(item))
                    try:
                        sys.path.insert(0, os.path.abspath(dir))
                        module = __import__(toks[0], globals(), locals())
                        if self._identify_plugin(module):
                            readers_found += 1
                    except :
                        logging.error("Loader: Error importing %s\n  %s" % (item, sys.exc_value))
                            
                # Process zip files
                elif item.endswith('.zip'):
                    try:
                        # Find the modules in the zip file
                        zfile = ZipFile(item)
                        nlist = zfile.namelist()
                        
                        sys.path.insert(0, item)
                        for mfile in nlist:
                            try:
                                # Change OS path to python path
                                fullname = mfile.replace('/', '.')
                                fullname = os.path.splitext(fullname)[0]
                                module = __import__(fullname, globals(), locals(), [""])
                                if self._identify_plugin(module):
                                    readers_found += 1
                            except:
                                logging.error("Loader: Error importing %s\n  %s" % (mfile, sys.exc_value))
                            
                    except:
                        logging.error("Loader: Error importing %s\n  %s" % (item, sys.exc_value))
                     
        return readers_found 
    
    def _identify_plugin(self, module):
        """
            Look into a module to find whether it contains a 
            Reader class. If so, add it to readers and (potentially)
            to the list of writers.
            @param module: module object
        """
        reader_found = False
        
        if hasattr(module, "Reader"):
            try:
                # Find supported extensions
                loader = module.Reader()
                for ext in loader.ext:
                    if ext not in self.loaders:
                        self.loaders[ext] = []
                    #The first reading will use the reader with only one ext.
                    if len(self.loaders[ext])>0 and len(loader.ext)>2:
                        self.loaders[ext].insert(1,loader.read)
                    else:
                        self.loaders[ext].insert(0,loader.read)

                    reader_found = True
                    
                    # Keep track of wildcards
                    for wcard in loader.type:
                        if wcard not in self.wildcards:
                            self.wildcards.append(wcard)
                            
                # Check whether writing is supported
                if hasattr(loader, 'write'):
                    for ext in loader.ext:
                        if ext not in self.writers:
                            self.writers[ext] = []
                        self.writers[ext].insert(0,loader.write)
                        
            except:
                logging.error("Loader: Error accessing Reader in %s\n  %s" % (name, sys.exc_value))
        return reader_found

    def lookup_writers(self, path):
        """
        Return the loader associated with the file type of path.
        
        Raises ValueError if file type is not known.
        """        
{'fraction_non_alphanumeric': 0.07274937060508725, 'fraction_numerical': 0.0025175796510113727, 'mean_word_length': 1.2916252237915258, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4078036', 'n_tokens_mistral': 2820, 'n_tokens_neox': 2647, 'n_words': 895}