<commit_before>"""
Lower level mixins.

"""
import os
import yaml
import sys
import cPickle
import logging
from multiprocessing import Process

class ContainerCore(object):
    """Mixin class for all Containers.

    The ContainerCore object is not intended to be useful on its own, but
    instead contains methods and attributes common to all Container objects.

    """
    def __init__(self):
        """
        
        """
        self.metadata = dict()              # information about object; defines base object
        self.analysis = dict()              # analysis data 'modular dock'
    
    def save(self, *args):
        """Save base object metadata and, if desired, analysis data instances.

        By providing names of loaded datasets as arguments, you can save the
        loaded versions of the data to their source files. This is useful if
        you have need to manually edit the data in a source file, as you can
        load it, make changes, then write it out.

        If no arguments are given, then no datasets are saved to their source
        files. Only metadata is saved.

        If 'all' is in argument list, every dataset that is loaded is written
        to its file.

        :Arguments:
            *args*
                datasets to load
            
        :Keywords:
            *force*
                if True, reload data even if already loaded; default False

        """
        basedir = self._rel2abspath(self.metadata["basedir"])
        self._makedirs(basedir)

        with open(os.path.join(basedir, self.metadata['metafile']), 'w') as f:
            yaml.dump(self.metadata, f)

        if 'all' in args:
            self._logger.info("Saving all loaded data into source files for '{}'...".format(self.metadata['name']))
            for i in self.analysis:
                self._logger.info("Saving {}...".format(i))
                with open(os.path.join(self._rel2abspath(self.metadata['basedir']), '{}/{}.pkl'.format(i, i)), 'wb') as f:
                    cPickle.dump(self.analysis[i], f)
            self._logger.info("All loaded data saved.")
        elif len(args) != 0:
            self._logger.info("Saving selected data into source files for '{}'...".format(self.metadata['name']))
            for i in args:
                self._logger.info("Saving {}...".format(i))
                with open(os.path.join(self._rel2abspath(self.metadata['basedir']), '{}/{}.pkl'.format(i, i)), 'wb') as f:
                    cPickle.dump(self.analysis[i], f)
            self._logger.info("All selected data saved.")

    def refresh(self):
        """Reloads metadata from file.

        """
        basedir = self._rel2abspath(self.metadata['basedir'])
        metafile = os.path.join(basedir, self.metadata['metafile'])
        with open(metafile, 'r') as f:
            self.metadata = yaml.load(f)

    def load(self, *args, **kwargs):
        """Load data instances into object.

        If 'all' is in argument list, every available dataset is loaded.

        :Arguments:
            *args*
                datasets to load
            
        :Keywords:
            *force*
                if True, reload data even if already loaded; default False
        """

        force = kwargs.pop('force', False)

        if 'all' in args:
            self._logger.info("Loading all known data into object '{}'...".format(self.metadata['name']))
            loadlist = self.metadata['analyses']
        else:
            self._logger.info("Loading selected data into object '{}'...".format(self.metadata['name']))
            loadlist = args

        for i in loadlist:
            if (i not in self.analysis) or (force == True):
                self._logger.info("Loading {}...".format(i))
                with open(os.path.join(self._rel2abspath(self.metadata['basedir']), '{}/{}.pkl'.format(i, i)), 'rb') as f:
                    self.analysis[i] = cPickle.load(f)
            else:
                self._logger.info("Skipping reload of {}...".format(i))
            self._logger.info("Object '{}' loaded with selected data.".format(self.metadata['name']))

    def unload(self, *args):
        """Unload data instances from object.<commit_msg>Refactored ContainerCore.save.<commit_after>"""
Lower level mixins.

"""
import os
import yaml
import sys
import cPickle
import logging
from multiprocessing import Process

class ContainerCore(object):
    """Mixin class for all Containers.

    The ContainerCore object is not intended to be useful on its own, but
    instead contains methods and attributes common to all Container objects.

    """
    def __init__(self):
        """
        
        """
        self.metadata = dict()              # information about object; defines base object
        self.analysis = dict()              # analysis data 'modular dock'
    
    def save(self, *args):
        """Save base object metadata and, if desired, analysis data instances.

        By providing names of loaded datasets as arguments, you can save the
        loaded versions of the data to their source files. This is useful if
        you have need to manually edit the data in a source file, as you can
        load it, make changes, then write it out.

        If no arguments are given, then no datasets are saved to their source
        files. Only metadata is saved.

        If 'all' is in argument list, every dataset that is loaded is written
        to its file.

        :Arguments:
            *args*
                datasets to load
            
        :Keywords:
            *force*
                if True, reload data even if already loaded; default False

        """
        basedir = self._rel2abspath(self.metadata["basedir"])
        self._makedirs(basedir)

        with open(os.path.join(basedir, self.metadata['metafile']), 'w') as f:
            yaml.dump(self.metadata, f)

        if 'all' in args:
            self._logger.info("Saving all loaded data into source files for '{}'...".format(self.metadata['name']))
            savelist = self.analysis
        elif len(args) != 0:
            self._logger.info("Saving selected data into source files for '{}'...".format(self.metadata['name']))
            savelist = args

        for i in savelist:
            self._logger.info("Saving {}...".format(i))
            with open(os.path.join(self._rel2abspath(self.metadata['basedir']), '{}/{}.pkl'.format(i, i)), 'wb') as f:
                cPickle.dump(self.analysis[i], f)
        self._logger.info("All selected data saved.")

    def refresh(self):
        """Reloads metadata from file.

        """
        basedir = self._rel2abspath(self.metadata['basedir'])
        metafile = os.path.join(basedir, self.metadata['metafile'])
        with open(metafile, 'r') as f:
            self.metadata = yaml.load(f)

    def load(self, *args, **kwargs):
        """Load data instances into object.

        If 'all' is in argument list, every available dataset is loaded.

        :Arguments:
            *args*
                datasets to load
            
        :Keywords:
            *force*
                if True, reload data even if already loaded; default False
        """

        force = kwargs.pop('force', False)

        if 'all' in args:
            self._logger.info("Loading all known data into object '{}'...".format(self.metadata['name']))
            loadlist = self.metadata['analyses']
        else:
            self._logger.info("Loading selected data into object '{}'...".format(self.metadata['name']))
            loadlist = args

        for i in loadlist:
            if (i not in self.analysis) or (force == True):
                self._logger.info("Loading {}...".format(i))
                with open(os.path.join(self._rel2abspath(self.metadata['basedir']), '{}/{}.pkl'.format(i, i)), 'rb') as f:
                    self.analysis[i] = cPickle.load(f)
            else:
                self._logger.info("Skipping reload of {}...".format(i))
        self._logger.info("Object '{}' loaded with selected data.".format(self.metadata['name']))

    def unload(self, *args):
        """Unload data instances from object.
{'fraction_non_alphanumeric': 0.1205795677799607, 'fraction_numerical': 0.0013506876227897839, 'mean_word_length': 2.0437219730941703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5788438', 'n_tokens_mistral': 2244, 'n_tokens_neox': 2165, 'n_words': 725}