<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------
# Name: cache_file.py
# Python Library
# Author: <NAME>
# Purpose: Persistant file-backed cache using /tmp/ to share data
#          using flock or msvcrt.locking to allow safe concurrent
#          access.
#-----------------------

import struct
import errno
import json
import time
import os

from cache_engine import CacheEngine

def _donothing(*args, **kwargs):
    pass

try:
    import fcntl
    class Flock( object ):
        """
        Context manager to flock file for the duration the object exists.
        Referenced file will be automatically unflocked as the interpreter
        exits the context.
        Supports an optional callback to process the error and optionally
        suppress it.
        """
        LOCK_EX = fcntl.LOCK_EX
        LOCK_SH = fcntl.LOCK_SH

        def __init__(self, fileobj, operation, callback=None):
            self.fileobj = fileobj
            self.operation = operation
            self.callback = callback
        def __enter__(self):
            fcntl.flock(self.fileobj, self.operation)
        def __exit__(self, exc_type, exc_value, exc_tb):
            suppress = False
            if callable(self.callback):
                suppress = self.callback(exc_type, exc_value, exc_tb)
            fcntl.flock(self.fileobj, fcntl.LOCK_UN)
            return suppress

    def parse_filename(filename):
        if '$' in filename:
            # replace any environmental variables
            filename = os.path.expandvars(filename)
        if filename.startswith('~'):
            # check for home directory
            return os.path.expanduser(filename)
        elif not filename.startswith('/'):
            # check for absolute path
            return filename
        # return path with temp directory prepended
        return '/tmp/' + filename

except ImportError:
    import msvcrt
    class Flock( object ):
        LOCK_EX = msvcrt.LK_LOCK
        LOCK_SH = msvcrt.LK_LOCK

        def __init__(self, fileobj, operation, callback=None):
            self.fileobj = fileobj
            self.operation = operation
            self.callback = callback
        def __enter__(self):
            self.size = os.path.getsize(self.fileobj.name)
            msvcrt.locking(self.fileobj.fileno(), self.operation, self.size)
        def __exit__(self, exc_type, exc_value, exc_tb):
            suppress = False
            if callable(self.callback):
                suppress = self.callback(exc_type, exc_value, exc_tb)
            msvcrt.locking(self.fileobj.fileno(), msvcrt.LK_UNLCK, self.size)
            return suppress
<commit_msg>Correct cache file selection logic inverted by 4663490f8fa.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#-----------------------
# Name: cache_file.py
# Python Library
# Author: <NAME>
# Purpose: Persistant file-backed cache using /tmp/ to share data
#          using flock or msvcrt.locking to allow safe concurrent
#          access.
#-----------------------

import struct
import errno
import json
import time
import os

from cache_engine import CacheEngine

def _donothing(*args, **kwargs):
    pass

try:
    import fcntl
    class Flock( object ):
        """
        Context manager to flock file for the duration the object exists.
        Referenced file will be automatically unflocked as the interpreter
        exits the context.
        Supports an optional callback to process the error and optionally
        suppress it.
        """
        LOCK_EX = fcntl.LOCK_EX
        LOCK_SH = fcntl.LOCK_SH

        def __init__(self, fileobj, operation, callback=None):
            self.fileobj = fileobj
            self.operation = operation
            self.callback = callback
        def __enter__(self):
            fcntl.flock(self.fileobj, self.operation)
        def __exit__(self, exc_type, exc_value, exc_tb):
            suppress = False
            if callable(self.callback):
                suppress = self.callback(exc_type, exc_value, exc_tb)
            fcntl.flock(self.fileobj, fcntl.LOCK_UN)
            return suppress

    def parse_filename(filename):
        if '$' in filename:
            # replace any environmental variables
            filename = os.path.expandvars(filename)
        if filename.startswith('~'):
            # check for home directory
            return os.path.expanduser(filename)
        elif filename.startswith('/'):
            # check for absolute path
            return filename
        # return path with temp directory prepended
        return '/tmp/' + filename

except ImportError:
    import msvcrt
    class Flock( object ):
        LOCK_EX = msvcrt.LK_LOCK
        LOCK_SH = msvcrt.LK_LOCK

        def __init__(self, fileobj, operation, callback=None):
            self.fileobj = fileobj
            self.operation = operation
            self.callback = callback
        def __enter__(self):
            self.size = os.path.getsize(self.fileobj.name)
            msvcrt.locking(self.fileobj.fileno(), self.operation, self.size)
        def __exit__(self, exc_type, exc_value, exc_tb):
            suppress = False
            if callable(self.callback):
                suppress = self.callback(exc_type, exc_value, exc_tb)
            msvcrt.locking(self.fileobj.fileno(), msvcrt.LK_UNLCK, self.size)
            return suppress

{'fraction_non_alphanumeric': 0.09699554896142433, 'fraction_numerical': 0.00185459940652819, 'mean_word_length': 2.03318335208099, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4781940', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1465, 'n_words': 420}