<commit_before># -*- encoding: utf-8 -*-

import errno
import io
import os
import re
import tempfile
from binascii import hexlify
from functools import wraps
from hashlib import md5
from time import time

from tagcache.lock import FileLock
from tagcache.serialize import PickleSerializer
from tagcache.utils import cached_property, link_file, rename_file, \
        silent_close, silent_unlink, ensure_dir


# datetime.fromtimestamp(2**32)  -> datetime.datetime(2106, 2, 7, 14, 28, 16)
_future_timestamp = 2**32


class Cache(object):
    """

    'key' and 'tags' can only contains [a-zA-Z0-9\-_\.@].

    """

    key_matcher = re.compile(r'^[A-z0-9\-_\.@]+$').match

    def __init__(self, hash_method=md5, serializer=None):
        """
        Create a cache object. `hash_method` is used to hash keys
        into file path; `serializer` is used to dump/load object
        to/from io
        
        """

        self.hash_method = hash_method

        if serializer is None:

            serializer = PickleSerializer()

        self.serializer = serializer

    def configure(self, main_dir):
        """
        Configure the cache.

        :param main_dir: the dir contains everything.

        """

        main_dir = os.path.abspath(main_dir)

        if not os.path.isdir(main_dir):

            raise ValueError("{0} is not a directory".format(main_dir))

        self.main_dir = main_dir

    @cached_property
    def data_dir(self):

        ret = os.path.join(self.main_dir, 'data')

        ensure_dir(ret)

        return ret

    @cached_property
    def tmp_dir(self):

        ret = os.path.join(self.main_dir, 'tmp')

        ensure_dir(ret)

        return ret

    def name_to_path(self, name, ns=''):

        prefix = '' if not ns else ns + ':'

        name = prefix + hexlify(name)

        h = self.hash_method(name).hexdigest()

        return os.path.join(self.data_dir, h[:2], h[2:4], name)

    def key_to_path(self, name):

        return self.name_to_path(name, ns='key')

    def tag_to_path(self, name):

        return self.name_to_path(name, ns='tag')

    def invalidate_tag(self, tag):
        """
        Invalidate cache with the tag.

        """
        if not self.key_matcher(tag):

            raise ValueError("Bad tag format {0!r}".format(tag))

        silent_unlink(self.tag_to_path(tag))

    def invalidate_key(self, key):
        """
        Invalidate cache with the key.

        """

        if not self.key_matcher(key):

            raise ValueError("Bad key format {0!r}".format(key))

        silent_unlink(self.key_to_path(key))
<commit_msg>os.unlink can not unlink dir.<commit_after># -*- encoding: utf-8 -*-

import errno
import io
import os
import re
import shutil
import tempfile
from binascii import hexlify
from functools import wraps
from hashlib import md5
from time import time

from tagcache.lock import FileLock
from tagcache.serialize import PickleSerializer
from tagcache.utils import cached_property, link_file, rename_file, \
        silent_close, silent_unlink, ensure_dir


# datetime.fromtimestamp(2**32)  -> datetime.datetime(2106, 2, 7, 14, 28, 16)
_future_timestamp = 2**32


class Cache(object):
    """

    'key' and 'tags' can only contains [a-zA-Z0-9\-_\.@].

    """

    key_matcher = re.compile(r'^[<KEY>').match

    def __init__(self, hash_method=md5, serializer=None):
        """
        Create a cache object. `hash_method` is used to hash keys
        into file path; `serializer` is used to dump/load object
        to/from io
        
        """

        self.hash_method = hash_method

        if serializer is None:

            serializer = PickleSerializer()

        self.serializer = serializer

    def configure(self, main_dir):
        """
        Configure the cache.

        :param main_dir: the dir contains everything.

        """

        main_dir = os.path.abspath(main_dir)

        if not os.path.isdir(main_dir):

            raise ValueError("{0} is not a directory".format(main_dir))

        self.main_dir = main_dir

    @cached_property
    def data_dir(self):

        ret = os.path.join(self.main_dir, 'data')

        ensure_dir(ret)

        return ret

    @cached_property
    def tmp_dir(self):

        ret = os.path.join(self.main_dir, 'tmp')

        ensure_dir(ret)

        return ret

    def name_to_path(self, name, ns=''):

        prefix = '' if not ns else ns + ':'

        name = prefix + hexlify(name)

        h = self.hash_method(name).hexdigest()

        return os.path.join(self.data_dir, h[:2], h[2:4], name)

    def key_to_path(self, name):

        return self.name_to_path(name, ns='key')

    def tag_to_path(self, name):

        return self.name_to_path(name, ns='tag')

    def invalidate_tag(self, tag):
        """
        Invalidate cache with the tag.

        """
        if not self.key_matcher(tag):

            raise ValueError("Bad tag format {0!r}".format(tag))

        shutil.rmtree(self.tag_to_path(tag), ignore_errors=True)

    def invalidate_key(self, key):
        """
        Invalidate cache with the key.

        """

        if not self.key_matcher(key):

            raise ValueError("Bad key format {0!r}".format(key))

        silent_unlink(self.key_to_path(key))

{'fraction_non_alphanumeric': 0.11324704986676817, 'fraction_numerical': 0.011419870574800152, 'mean_word_length': 2.392511297611362, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1602319', 'n_tokens_mistral': 1760, 'n_tokens_neox': 1645, 'n_words': 427}