<commit_before>import os
import inspect
import logging
log = logging.getLogger(__file__)

from ott.utils import file_utils
from ott.utils import web_utils
from ott.utils.config_util import ConfigUtil


class CacheBase(object):
    cache_dir_name = "cache"
    cache_expire = 31
    _config = None

    def __init__(self, section="cache", cache_dir_name="cache"):
        self._config = ConfigUtil.factory(section=section)
        self.cache_expire = self.config.get_int('cache_expire', 'cache', self.cache_expire)
        self.cache_dir_name = cache_dir_name

    @property
    def config(self):
        #import pdb; pdb.set_trace()
        if self._config is None:
            self._config = ConfigUtil.factory(section='cache')
        return self._config

    @property
    def this_module_dir(self):
        ''' set object property 'this_module_dir' to the file directory where the 'self' object lives
        '''
        file = inspect.getsourcefile(self.__class__)
        dir = os.path.dirname(os.path.abspath(file))
        return dir

    @property
    def cache_dir(self):
        ''' returns dir path ... makes the directory if it doesn't exist
        '''
        cache_dir = os.path.join(self.this_module_dir, self.cache_dir_name)
        file_utils.mkdir(cache_dir)
        return cache_dir

    @property
    def tmp_dir(self):
        tmp_dir = os.path.join(self.this_module_dir, "tmp")
        file_utils.mkdir(tmp_dir)
        return tmp_dir

    def is_fresh_in_cache(self, file):
        ''' determine if file exists and is newer than the cache expire time
        '''
        ret_val = False
        try:
            # NOTE if the file isn't in the cache, we'll get an exception
            age = file_utils.file_age(file)
            if age < self.cache_expire:
                ret_val = True
        except:
            ret_val = False
        return ret_val

    def cp_cached_file(self, file_name, destination_dir):
        ''' copy file from our cache to some other directory
        '''
        file = os.path.join(self.cache_dir, file_name)
        dest = os.path.join(destination_dir, file_name)
        file_utils.cp(file, dest)

    def simple_cache_item_update(self, file_name, url, force_update=False):
        ''' check a cache file to see whether it needs updating...if so,
            then replace cached file with new version via wget on a  url
        '''
        file_path = os.path.join(self.cache_dir, file_name)

        # step 1: check the cache whether we should update or not
        update = force_update
        if not force_update and not self.is_fresh_in_cache(file_path):
            update = True

        # step 2: backup then wget new feed
        if update:
            log.info("wget {} to cache {}".format(url, file_path))
            file_utils.bkup(file_path)
            web_utils.wget(url, file_path)

        return update
<commit_msg>new dir utility<commit_after>import os
import inspect
import logging
log = logging.getLogger(__file__)

from ott.utils import file_utils
from ott.utils import web_utils
from ott.utils.config_util import ConfigUtil


class CacheBase(object):
    cache_dir_name = "cache"
    cache_expire = 31
    _config = None

    def __init__(self, section="cache", cache_dir_name="cache"):
        self._config = ConfigUtil.factory(section=section)
        self.cache_expire = self.config.get_int('cache_expire', 'cache', self.cache_expire)
        self.cache_dir_name = cache_dir_name

    @property
    def config(self):
        #import pdb; pdb.set_trace()
        if self._config is None:
            self._config = ConfigUtil.factory(section='cache')
        return self._config

    @property
    def this_module_dir(self):
        ''' set object property 'this_module_dir' to the file directory where the 'self' object lives
        '''
        file = inspect.getsourcefile(self.__class__)
        dir = os.path.dirname(os.path.abspath(file))
        return dir

    @property
    def cache_dir(self):
        ''' returns dir path ... makes the directory if it doesn't exist
        '''
        cache_dir = os.path.join(self.this_module_dir, self.cache_dir_name)
        file_utils.mkdir(cache_dir)
        return cache_dir

    @property
    def tmp_dir(self):
        tmp_dir = os.path.join(self.this_module_dir, "tmp")
        file_utils.mkdir(tmp_dir)
        return tmp_dir

    def sub_dir(self, dir_name):
        '''make path to random named dir
        '''
        ret_val = os.path.join(self.this_module_dir, dir_name)
        return ret_val

    def is_fresh_in_cache(self, file):
        ''' determine if file exists and is newer than the cache expire time
        '''
        ret_val = False
        try:
            # NOTE if the file isn't in the cache, we'll get an exception
            age = file_utils.file_age(file)
            if age < self.cache_expire:
                ret_val = True
        except:
            ret_val = False
        return ret_val

    def cp_cached_file(self, file_name, destination_dir):
        ''' copy file from our cache to some other directory
        '''
        file = os.path.join(self.cache_dir, file_name)
        dest = os.path.join(destination_dir, file_name)
        file_utils.cp(file, dest)

    def simple_cache_item_update(self, file_name, url, force_update=False):
        ''' check a cache file to see whether it needs updating...if so,
            then replace cached file with new version via wget on a  url
        '''
        file_path = os.path.join(self.cache_dir, file_name)

        # step 1: check the cache whether we should update or not
        update = force_update
        if not force_update and not self.is_fresh_in_cache(file_path):
            update = True

        # step 2: backup then wget new feed
        if update:
            log.info("wget {} to cache {}".format(url, file_path))
            file_utils.bkup(file_path)
            web_utils.wget(url, file_path)

        return update

{'fraction_non_alphanumeric': 0.08633696563285834, 'fraction_numerical': 0.001341156747694887, 'mean_word_length': 2.384004537719796, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1967628', 'n_tokens_mistral': 1787, 'n_tokens_neox': 1741, 'n_words': 492}