<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Cached file-system utility library
# (C) 2016 VRT Systems
#
# vim: set ts=4 sts=4 et tw=78 sw=4 si:

"""
cachefs: a cached filesystem abstraction library.

The purpose of this library is to provide a simple cache buffer for file system
operations.  Sometimes in a process you might need to know whether a file
changed, or what its permissions are, or its size, and you might need to know
in multiple places.  These different pieces of code may be completely oblivious
to each-other, and so rather than re-using the same result, you call `os.stat`
twice.

The concept here is to provide a way to cache filesystem metadata, for
situations where you know the files themselves won't change that frequently,
but you know you'll be needing to refer back to this information a lot.
"""

import time
import os
import weakref

from .node import Node
from pyat.base import TaskScheduler
from pyat.sync import SynchronousTaskScheduler

class CacheFs(object):
    '''
    A cached filesystem instance.  This holds strong references to nodes that
    are being frequently accessed by the end user.
    '''

    def __init__(self, cache_expiry, stat_expiry, scheduler=None):
        # node cache expiry
        self._cache_expiry = float(cache_expiry)

        # stat() expiry duration
        self._stat_expiry = float(stat_expiry)

        # Active nodes.
        self._nodes = {}

        # Scheduler instance.
        if scheduler is None:
            scheduler = SynchronousTaskScheduler()
        elif not isinstance(scheduler, TaskScheduler):
            raise TypeError('%r is not a TaskScheduler instance' % scheduler)
        self._scheduler = scheduler
        self._purge_task = None

    @property
    def _required_time(self):
        return time.time() - self._stat_expiry

    @property
    def _min_atime(self):
        return min([node._atime for node in self._nodes.values()])

    def _purge(self):
        '''
        Purge the cache of old entries.
        '''
        for n in filter(lambda n : n.atime_since > self._cache_expiry,
                self._nodes.values()):
            try:
                del self._nodes[n.abs_path]
            except KeyError:  # pragma: no cover<commit_msg>cachefs: Convert generators to lists for Python 3.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Cached file-system utility library
# (C) 2016 VRT Systems
#
# vim: set ts=4 sts=4 et tw=78 sw=4 si:

"""
cachefs: a cached filesystem abstraction library.

The purpose of this library is to provide a simple cache buffer for file system
operations.  Sometimes in a process you might need to know whether a file
changed, or what its permissions are, or its size, and you might need to know
in multiple places.  These different pieces of code may be completely oblivious
to each-other, and so rather than re-using the same result, you call `os.stat`
twice.

The concept here is to provide a way to cache filesystem metadata, for
situations where you know the files themselves won't change that frequently,
but you know you'll be needing to refer back to this information a lot.
"""

import time
import os
import weakref

from .node import Node
from pyat.base import TaskScheduler
from pyat.sync import SynchronousTaskScheduler

class CacheFs(object):
    '''
    A cached filesystem instance.  This holds strong references to nodes that
    are being frequently accessed by the end user.
    '''

    def __init__(self, cache_expiry, stat_expiry, scheduler=None):
        # node cache expiry
        self._cache_expiry = float(cache_expiry)

        # stat() expiry duration
        self._stat_expiry = float(stat_expiry)

        # Active nodes.
        self._nodes = {}

        # Scheduler instance.
        if scheduler is None:
            scheduler = SynchronousTaskScheduler()
        elif not isinstance(scheduler, TaskScheduler):
            raise TypeError('%r is not a TaskScheduler instance' % scheduler)
        self._scheduler = scheduler
        self._purge_task = None

    @property
    def _required_time(self):
        return time.time() - self._stat_expiry

    @property
    def _min_atime(self):
        return min([node._atime for node in self._nodes.values()])

    def _purge(self):
        '''
        Purge the cache of old entries.
        '''
        all_nodes = list(self._nodes.values())
        expired = list(filter(lambda n : n.atime_since > self._cache_expiry,
            all_nodes))
        for n in expired:
            try:
                del self._nodes[n.abs_path]
            except KeyError:  # pragma: no cover
{'fraction_non_alphanumeric': 0.07404200043299415, 'fraction_numerical': 0.004546438623078589, 'mean_word_length': 2.7962202136400984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5279204', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1265, 'n_words': 514}