<commit_before>"""
Classes for datafile syncronization. 

"""

import Aggregators
import Workers
from uuid import uuid4
import tables
import fcntl
import os
from functools import wraps

import pdb

class File(object):
    """File object base class. Implements file locking and reloading methods.
    
    """
    def __init__(self, filename, logger=None, **kwargs):
        """Create File instance for interacting with file on disk.

        All files in MDSynthesis should be accessible by high-level
        methods without having to worry about simultaneous reading and writing by
        other processes. The File object includes methods and infrastructure
        for ensuring shared and exclusive locks are consistently applied before
        reads and writes, respectively. It handles any other low-level tasks
        for maintaining file integrity.

        :Arguments:
           *filename*
              name of file on disk object corresponds to 
           *logger*
              logger to send warnings and errors to

        """
        self.filename = os.path.abspath(filename)
        self.handle = None
        self.logger = logger

    def shlock(self):
        """Get shared lock on file.

        Using fcntl.lockf, a shared lock on the file is obtained. If an
        exclusive lock is already held on the file by another process,
        then the method waits until it can obtain the lock.

        :Returns:
           *success*
              True if shared lock successfully obtained
        """
        fcntl.lockf(self.handle, fcntl.LOCK_SH)

        return True

    def exlock(self):
        """Get exclusive lock on file.
    
        Using fcntl.lockf, an exclusive lock on the file is obtained. If a
        shared or exclusive lock is already held on the file by another
        process, then the method waits until it can obtain the lock.

        :Returns:
           *success*
              True if exclusive lock successfully obtained
        """
        # first obtain shared lock; may help to avoid race conditions between
        # exclusive locks (REQUIRES THOROUGH TESTING)
        if self.shlock():
            fcntl.lockf(self.handle, fcntl.LOCK_EX)
    
        return True

    def unlock(self):
        """Remove exclusive or shared lock on file.

        :Returns:
           *success*
              True if lock removed
        """
        fcntl.lockf(self.handle, fcntl.LOCK_UN)

        return True

    def check_existence(self):
        """Check for existence of file.
    
        """
        return os.path.exists(self.filename)

class ContainerFile(File):
    """Container file object; syncronized access to Container data.

    """
    class Meta(tables.IsDescription):
        """Table definition for metadata.

        All strings limited to hardcoded size for now.

        """
        # unique identifier for container
        uuid = tables.StringCol(36)

        # user-given name of container
        name = tables.StringCol(128)

        # container type; Sim or Group
        container = tables.StringCol(36)

        # eventually we would like this to be generated dynamically
        # meaning, size of location string is size needed, and meta table
        # is regenerated if any of its strings need to be (or smaller)
        # When Coordinator generates its database, it uses largest string size
        # needed
        location = tables.StringCol(256)

    class Coordinator(tables.IsDescription):
        """Table definition for coordinator info.

        This information is kept separate from other metadata to allow the
        Coordinator to simply stack tables to populate its database. It doesn't
        need entries that store its own path.

        Path length fixed size for now.
        """
        # absolute path of coordinator
        abspath = tables.StringCol(256)
        
    class Tags(tables.IsDescription):
        """Table definition for tags.

        """
        tag = tables.StringCol(36)

    class Categories(tables.IsDescription):
        """Table definition for categories.

        """
        category = tables.StringCol(36)
        value = tables.StringCol(36)

    def __init__(self, filename, logger, classname, **kwargs): 
        """Initialize Container state file.

        This is the base class for all Container state files. It generates 
        data structure elements common to all Containers. It also implements
        low-level I/O functionality.

        :Arguments:
           *filename*
              path to file
           *logger*
              Container's logger instance
           *classname*
              Container's class name

        :Keywords:
           *name*
              user-given name of Container object
           *coordinator*
              directory in which coordinator state file can be found [None]
           *categories*
              user-given dictionary with custom keys and values; used to
              give distinguishing characteristics to object for search
           *tags*
              user-given list with custom elements; used to give distinguishing
              characteristics to object for search
           *details*
              user-given string for object notes

        .. Note:: kwargs passed to :meth:`create`

        """
        super(ContainerFile, self).__init__(filename, logger=logger)
        
        # if file does not exist, it is created
        if not self.check_existence():
            self.create(classname, **kwargs)

    def create(self, classname, **kwargs):
        """Build state file and common data structure elements.

        :Arguments:
           *classname*
              Container's class name

        :Keywords:
           *name*
              user-given name of Container object
           *coordinator*
              directory in which coordinator state file can be found [None]
           *categories*
              user-given dictionary with custom keys and values; used to
              give distinguishing characteristics to object for search
           *tags*
              user-given list with custom elements; used to give distinguishing
              characteristics to object for search
        """
        self.handle = tables.open_file(self.filename, 'w')
        self.exlock()

        # metadata table
        meta_table = self.handle.create_table('/', 'meta', self.Meta, 'metadata')
        container = meta_table.row

        container['uuid'] = str(uuid4())
        container['name'] = kwargs.pop('name', classname)
        container['container'] = classname
        container['location'] = os.path.dirname(self.filename)
        container.append()

        # coordinator table
        coordinator_table = self.handle.create_table('/', 'coordinator', self.Coordinator, 'coordinator information')
        container = coordinator_table.row
        
        container['abspath'] = kwargs.pop('coordinator', None)
        container.append()

        # tags table
        #TODO: make use of add_tags methods, but must be wary of multiple opens to file
        # previous opens are only closed when the interpreter exits
        tags_table = self.handle.create_table('/', 'tags', self.Tags, 'tags')
        container = tags_table.row
        
        tags = kwargs.pop('tags', list())
        for tag in tags:
            container['tag'] = str(tag)
            container.append()

        # categories table
        categories_table = self.handle.create_table('/', 'categories', self.Categories, 'categories')
        container = categories_table.row
        
        categories = kwargs.pop('categories', dict())
        for key in categories.keys():
            container['category'] = str(key)
            container['value'] = str(categories[key])
            container.append()

        # remove lock and close
        self.unlock()
        self.handle.close()
    
    def read(self, func):
        """Decorator for opening file for reading and applying shared lock.
        
        Applying this decorator to a method will ensure that the file is opened
        for reading and that a shared lock is obtained before that method is
        executed. It also ensures that the lock is removed and the file closed
        after the method returns.

        """
        @wraps(func)
        def inner(*args, **kwargs):
            self.handle = tables.open_file(self.filename, 'r')
            self.shlock()
            func(*args, **kwargs)
            self.unlock()
            self.handle.close()

        return inner
    
    def write(func):
        """Decorator for opening file for writing and applying exclusive lock.
        
        Applying this decorator to a method will ensure that the file is opened
        for appending and that an exclusive lock is obtained before that method is
        executed. It also ensures that the lock is removed and the file closed
        after the method returns.

        """
        @wraps(func)
        def inner(self, *args, **kwargs):
            self.handle = tables.open_file(self.filename, 'a')
            self.exlock()
            func(self, *args, **kwargs)
            self.unlock()
            self.handle.close()

        return inner

    @write
    def add_tags(self, *tags):
        """Add any number of tags to the Container.

        Tags are individual strings that serve to differentiate Containers from
        one another. Sometimes preferable to categories.

        :Arguments:
           *tags*
              Tags to add. Must be convertable to strings using the str() builtin.

        """
        tags_table = self.handle.get_node('/', 'tags')
        container = tags_table.row

        # ensure tags are unique (we don't care about order)
        tags = set(tags)

        # remove tags already present in metadata from list
        #TODO: more efficient way to do this?
        tags_present = list()
        for tag in tags:
            for row in tags_table:
                if (row['tag'] == str(tag)):
                    tags_present.append(tag)

        tags = list(tags - set(tags_present))

        # add new tags
        for tag in tags:
            container['tag'] = str(tag)
            container.append()

    @write
    def add_categories(self, **categories):
        """Add any number of categories to the Container.

        Categories are key-value pairs of strings that serve to differentiate
        Containers from one another. Sometimes preferable to tags.

        :Keywords:
            *categories*
                Categories to add. Keyword used as key, value used as value. Both
                must be convertible to strings using the str() builtin.

        """
        categories_table = self.handle.get_node('/', 'categories')
        container = categories_table.row

        # remove categories already present in metadata from dictionary 
        #TODO: more efficient way to do this?
        for key in categories.keys():
            for row in categories_table:
                if (row['category'] == str(key)):
                    row['value'] = str(categories[key])
                    # dangerous? or not since we are iterating through
                    # categories.keys() and not categories?
                    categories.pop(key)
        
        # add new categories
        for key in categories.keys():
            container['category'] = str(key)
            container['value'] = str(categories[key])
            container.append()

    def _open_r(self):
        """Open file with intention to write.

        Not to be used except for debugging files.

        """
        self.handle = tables.open_file(self.filename, 'r')
        self.shlock()

    def _open_w(self):
        """Open file with intention to write.
    
        Not to be used except for debugging files.
         
        """
        self.handle = tables.open_file(self.filename, 'a')
        self.exlock()
    
    def _close(self):
        """Close file.<commit_msg>Clarifying comments to add_categories docs.<commit_after>"""
Classes for datafile syncronization. 

"""

import Aggregators
import Workers
from uuid import uuid4
import tables
import fcntl
import os
from functools import wraps

import pdb

class File(object):
    """File object base class. Implements file locking and reloading methods.
    
    """
    def __init__(self, filename, logger=None, **kwargs):
        """Create File instance for interacting with file on disk.

        All files in MDSynthesis should be accessible by high-level
        methods without having to worry about simultaneous reading and writing by
        other processes. The File object includes methods and infrastructure
        for ensuring shared and exclusive locks are consistently applied before
        reads and writes, respectively. It handles any other low-level tasks
        for maintaining file integrity.

        :Arguments:
           *filename*
              name of file on disk object corresponds to 
           *logger*
              logger to send warnings and errors to

        """
        self.filename = os.path.abspath(filename)
        self.handle = None
        self.logger = logger

    def shlock(self):
        """Get shared lock on file.

        Using fcntl.lockf, a shared lock on the file is obtained. If an
        exclusive lock is already held on the file by another process,
        then the method waits until it can obtain the lock.

        :Returns:
           *success*
              True if shared lock successfully obtained
        """
        fcntl.lockf(self.handle, fcntl.LOCK_SH)

        return True

    def exlock(self):
        """Get exclusive lock on file.
    
        Using fcntl.lockf, an exclusive lock on the file is obtained. If a
        shared or exclusive lock is already held on the file by another
        process, then the method waits until it can obtain the lock.

        :Returns:
           *success*
              True if exclusive lock successfully obtained
        """
        # first obtain shared lock; may help to avoid race conditions between
        # exclusive locks (REQUIRES THOROUGH TESTING)
        if self.shlock():
            fcntl.lockf(self.handle, fcntl.LOCK_EX)
    
        return True

    def unlock(self):
        """Remove exclusive or shared lock on file.

        :Returns:
           *success*
              True if lock removed
        """
        fcntl.lockf(self.handle, fcntl.LOCK_UN)

        return True

    def check_existence(self):
        """Check for existence of file.
    
        """
        return os.path.exists(self.filename)

class ContainerFile(File):
    """Container file object; syncronized access to Container data.

    """
    class Meta(tables.IsDescription):
        """Table definition for metadata.

        All strings limited to hardcoded size for now.

        """
        # unique identifier for container
        uuid = tables.StringCol(36)

        # user-given name of container
        name = tables.StringCol(128)

        # container type; Sim or Group
        container = tables.StringCol(36)

        # eventually we would like this to be generated dynamically
        # meaning, size of location string is size needed, and meta table
        # is regenerated if any of its strings need to be (or smaller)
        # When Coordinator generates its database, it uses largest string size
        # needed
        location = tables.StringCol(256)

    class Coordinator(tables.IsDescription):
        """Table definition for coordinator info.

        This information is kept separate from other metadata to allow the
        Coordinator to simply stack tables to populate its database. It doesn't
        need entries that store its own path.

        Path length fixed size for now.
        """
        # absolute path of coordinator
        abspath = tables.StringCol(256)
        
    class Tags(tables.IsDescription):
        """Table definition for tags.

        """
        tag = tables.StringCol(36)

    class Categories(tables.IsDescription):
        """Table definition for categories.

        """
        category = tables.StringCol(36)
        value = tables.StringCol(36)

    def __init__(self, filename, logger, classname, **kwargs): 
        """Initialize Container state file.

        This is the base class for all Container state files. It generates 
        data structure elements common to all Containers. It also implements
        low-level I/O functionality.

        :Arguments:
           *filename*
              path to file
           *logger*
              Container's logger instance
           *classname*
              Container's class name

        :Keywords:
           *name*
              user-given name of Container object
           *coordinator*
              directory in which coordinator state file can be found [None]
           *categories*
              user-given dictionary with custom keys and values; used to
              give distinguishing characteristics to object for search
           *tags*
              user-given list with custom elements; used to give distinguishing
              characteristics to object for search
           *details*
              user-given string for object notes

        .. Note:: kwargs passed to :meth:`create`

        """
        super(ContainerFile, self).__init__(filename, logger=logger)
        
        # if file does not exist, it is created
        if not self.check_existence():
            self.create(classname, **kwargs)

    def create(self, classname, **kwargs):
        """Build state file and common data structure elements.

        :Arguments:
           *classname*
              Container's class name

        :Keywords:
           *name*
              user-given name of Container object
           *coordinator*
              directory in which coordinator state file can be found [None]
           *categories*
              user-given dictionary with custom keys and values; used to
              give distinguishing characteristics to object for search
           *tags*
              user-given list with custom elements; used to give distinguishing
              characteristics to object for search
        """
        self.handle = tables.open_file(self.filename, 'w')
        self.exlock()

        # metadata table
        meta_table = self.handle.create_table('/', 'meta', self.Meta, 'metadata')
        container = meta_table.row

        container['uuid'] = str(uuid4())
        container['name'] = kwargs.pop('name', classname)
        container['container'] = classname
        container['location'] = os.path.dirname(self.filename)
        container.append()

        # coordinator table
        coordinator_table = self.handle.create_table('/', 'coordinator', self.Coordinator, 'coordinator information')
        container = coordinator_table.row
        
        container['abspath'] = kwargs.pop('coordinator', None)
        container.append()

        # tags table
        #TODO: make use of add_tags methods, but must be wary of multiple opens to file
        # previous opens are only closed when the interpreter exits
        tags_table = self.handle.create_table('/', 'tags', self.Tags, 'tags')
        container = tags_table.row
        
        tags = kwargs.pop('tags', list())
        for tag in tags:
            container['tag'] = str(tag)
            container.append()

        # categories table
        categories_table = self.handle.create_table('/', 'categories', self.Categories, 'categories')
        container = categories_table.row
        
        categories = kwargs.pop('categories', dict())
        for key in categories.keys():
            container['category'] = str(key)
            container['value'] = str(categories[key])
            container.append()

        # remove lock and close
        self.unlock()
        self.handle.close()
    
    def read(self, func):
        """Decorator for opening file for reading and applying shared lock.
        
        Applying this decorator to a method will ensure that the file is opened
        for reading and that a shared lock is obtained before that method is
        executed. It also ensures that the lock is removed and the file closed
        after the method returns.

        """
        @wraps(func)
        def inner(*args, **kwargs):
            self.handle = tables.open_file(self.filename, 'r')
            self.shlock()
            func(*args, **kwargs)
            self.unlock()
            self.handle.close()

        return inner
    
    def write(func):
        """Decorator for opening file for writing and applying exclusive lock.
        
        Applying this decorator to a method will ensure that the file is opened
        for appending and that an exclusive lock is obtained before that method is
        executed. It also ensures that the lock is removed and the file closed
        after the method returns.

        """
        @wraps(func)
        def inner(self, *args, **kwargs):
            self.handle = tables.open_file(self.filename, 'a')
            self.exlock()
            func(self, *args, **kwargs)
            self.unlock()
            self.handle.close()

        return inner

    @write
    def add_tags(self, *tags):
        """Add any number of tags to the Container.

        Tags are individual strings that serve to differentiate Containers from
        one another. Sometimes preferable to categories.

        :Arguments:
           *tags*
              Tags to add. Must be convertable to strings using the str() builtin.

        """
        tags_table = self.handle.get_node('/', 'tags')
        container = tags_table.row

        # ensure tags are unique (we don't care about order)
        tags = set(tags)

        # remove tags already present in metadata from list
        #TODO: more efficient way to do this?
        tags_present = list()
        for tag in tags:
            for row in tags_table:
                if (row['tag'] == str(tag)):
                    tags_present.append(tag)

        tags = list(tags - set(tags_present))

        # add new tags
        for tag in tags:
            container['tag'] = str(tag)
            container.append()

    @write
    def add_categories(self, **categories):
        """Add any number of categories to the Container.

        Categories are key-value pairs of strings that serve to differentiate
        Containers from one another. Sometimes preferable to tags.

        If a given category already exists (same key), the value given will replace
        the value for that category.

        :Keywords:
            *categories*
                Categories to add. Keyword used as key, value used as value. Both
                must be convertible to strings using the str() builtin.

        """
        categories_table = self.handle.get_node('/', 'categories')
        container = categories_table.row

        # remove categories already present in metadata from dictionary 
        #TODO: more efficient way to do this?
        for key in categories.keys():
            for row in categories_table:
                if (row['category'] == str(key)):
                    row['value'] = str(categories[key])
                    row.update()
                    # dangerous? or not since we are iterating through
                    # categories.keys() and not categories?
                    categories.pop(key)
        
        # add new categories
        for key in categories.keys():
            container['category'] = str(key)
            container['value'] = str(categories[key])
            container.append()

    def _open_r(self):
        """Open file with intention to write.

        Not to be used except for debugging files.

        """
        self.handle = tables.open_file(self.filename, 'r')
        self.shlock()

    def _open_w(self):
        """Open file with intention to write.
    
        Not to be used except for debugging files.
         
        """
        self.handle = tables.open_file(self.filename, 'a')
        self.exlock()
    
    def _close(self):
        """Close file.
{'fraction_non_alphanumeric': 0.07625750362243842, 'fraction_numerical': 0.0017387704409025046, 'mean_word_length': 1.963199214916585, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4706360', 'n_tokens_mistral': 5944, 'n_tokens_neox': 5694, 'n_words': 2280}