<commit_before># -*- coding: utf-8 -*-

"""
Module :mod:`database` contains helper function for manipulating the databases
"""

import os
import sys
from abc import ABCMeta, abstractmethod
import glob
import collections
from datetime import datetime
import shutil
import logging
try:
    import cPickle as pickle
except ImportError:
    import pickle


from .config import get_ct_folder


__author__ = "<NAME> <<EMAIL>>"
__copyright__ = "3-clause BSD License"


__EXP_NAME__ = "Experiment"
__PARAMETERS__ = "Parameters"
__RESULTS__ = "Results"
__CONTEXT__ = "Context"


class Architecture(object):
    """
    ``Architecture``
    ================
    An ``Architecture`` is responsible for the organization of the experiments
    and logs on the file system.
    By default, clustertools is located in a folder named 'clustertools_data'
    (aka ct_folder) in the user home directory. The ct_folder is structured as
    followed:
        clustertools_data
        |- logs
        |- exp_XXX
        |   |- logs
        |   |- $result$
        |   |- $notifs$
        |- exp_YYY
            |...
    where `logs` is a folder containing the main logs (not the ones corresponding
    to the experiments).  exp_XXX is the folder related to experiment 'XXX'.
    """
    def __init__(self, ct_folder=None):
        if ct_folder is None:
            ct_folder = get_ct_folder()
        self.ct_folder = ct_folder

    def __repr__(self):
        return "{cls}(ct_folder={folder})".format(cls=self.__class__.__name__,
                                                  folder=repr(self.ct_folder))

    def get_basedir(self, exp_name):
        return os.path.join(self.ct_folder, "exp_%s"%exp_name)

    def load_experiment_names(self):
        res = []
        for path in glob.glob(os.path.join(self.ct_folder, "exp_*")):
            if not os.path.isdir(path):
                continue
            exp_name = os.path.basename(path)[4:]  # remove 'exp_'
            res.append(exp_name)
        return res

    def get_meta_log_file(self):
        log_folder = os.path.join(self.ct_folder, "logs")
        if not os.path.exists(log_folder):
            os.makedirs(log_folder)
        fname = "clustertools_%s.log" % (datetime.now().strftime("%B%Y"))
        return os.path.join(log_folder, fname)

    def erase_experiment(self, exp_name):
        logger = logging.getLogger("clustertools")
        try:
            path = self.get_basedir(exp_name)
            if os.path.exists(path):
                shutil.rmtree(path)
        except OSError as error:
            logger.warn("Trouble erasing the databases: %s" % error.message,
                        exc_info=True)


# ============================== STORAGE MANAGER ============================= #

class Storage(object):
    """
    ``Storage``
    ===========
    A ``Storage`` objects manages two databases:
    - "notifications" which contains the states of the computations
    - "results" which contains the results of the computations<commit_msg>FIX version errors (2.7 vs 3.5)<commit_after># -*- coding: utf-8 -*-

"""
Module :mod:`database` contains helper function for manipulating the databases
"""

import os
import sys
from abc import ABCMeta, abstractmethod
import glob
import collections
from datetime import datetime
import shutil
import logging
try:
    import cPickle as pickle
except ImportError:
    import pickle


from .config import get_ct_folder


__author__ = "<NAME> <<EMAIL>>"
__copyright__ = "3-clause BSD License"


__EXP_NAME__ = "Experiment"
__PARAMETERS__ = "Parameters"
__RESULTS__ = "Results"
__CONTEXT__ = "Context"


class Architecture(object):
    """
    ``Architecture``
    ================
    An ``Architecture`` is responsible for the organization of the experiments
    and logs on the file system.
    By default, clustertools is located in a folder named 'clustertools_data'
    (aka ct_folder) in the user home directory. The ct_folder is structured as
    followed:
        clustertools_data
        |- logs
        |- exp_XXX
        |   |- logs
        |   |- $result$
        |   |- $notifs$
        |- exp_YYY
            |...
    where `logs` is a folder containing the main logs (not the ones corresponding
    to the experiments).  exp_XXX is the folder related to experiment 'XXX'.
    """
    def __init__(self, ct_folder=None):
        if ct_folder is None:
            ct_folder = get_ct_folder()
        self.ct_folder = ct_folder

    def __repr__(self):
        return "{cls}(ct_folder={folder})".format(cls=self.__class__.__name__,
                                                  folder=repr(self.ct_folder))

    def get_basedir(self, exp_name):
        return os.path.join(self.ct_folder, "exp_%s"%exp_name)

    def load_experiment_names(self):
        res = []
        for path in glob.glob(os.path.join(self.ct_folder, "exp_*")):
            if not os.path.isdir(path):
                continue
            exp_name = os.path.basename(path)[4:]  # remove 'exp_'
            res.append(exp_name)
        return res

    def get_meta_log_file(self):
        log_folder = os.path.join(self.ct_folder, "logs")
        if not os.path.exists(log_folder):
            os.makedirs(log_folder)
        fname = "clustertools_%s.log" % (datetime.now().strftime("%B%Y"))
        return os.path.join(log_folder, fname)

    def erase_experiment(self, exp_name):
        logger = logging.getLogger("clustertools")
        try:
            path = self.get_basedir(exp_name)
            if os.path.exists(path):
                shutil.rmtree(path)
        except OSError as error:
            logger.warning("Trouble erasing the databases: {}"
                           "".format(repr(error)),
                           exc_info=True)


# ============================== STORAGE MANAGER ============================= #

class Storage(object):
    """
    ``Storage``
    ===========
    A ``Storage`` objects manages two databases:
    - "notifications" which contains the states of the computations
    - "results" which contains the results of the computations
{'fraction_non_alphanumeric': 0.12685337726523888, 'fraction_numerical': 0.0016474464579901153, 'mean_word_length': 2.511278195488722, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2323151', 'n_tokens_mistral': 1819, 'n_tokens_neox': 1741, 'n_words': 481}