<commit_before>import sys
import os
import time
import rez_filesys
import rez_metafile
from versions import *
from public_enums import *
from rez_exceptions import *



_g_caching_enabled = True
_g_memcached_server = os.getenv("REZ_MEMCACHED_SERVER") or "127.0.0.1:11211"


def _create_client():
    if not _g_caching_enabled:
        return None
    return memcache.Client([_g_memcached_server], cache_cas=True)


def print_cache_warning(msg):
    print >> sys.stderr, "Cache Warning: %s" % msg


# init
_g_caching_enabled = not os.getenv("REZ_DISABLE_CACHING")
if _g_caching_enabled:
    try:
        import memcache
        from memcached_client import *
    except:
        _g_caching_enabled = False
if _g_caching_enabled:
    mc = _create_client()
    if not mc.set("test_set", "success"):
        _g_caching_enabled = False
    mc = None


class RezMemCache(object):
    """
    Cache for filesystem access and resolves.
    """
    def __init__(self, time_epoch=0, use_caching=True):
        self.epoch = time_epoch or int(time.time())
        self.families = set()
        self.versions = {} # (path,order): [versions]
        self.metafiles = {} # path, ConfigMetadata
        self.mc = None
        if use_caching and _g_caching_enabled:
            mc = _create_client()
            self.mc = MemCacheClient(mc)

    def caching_enabled(self):
        """
        whether the memcache client is being used
        """
        return bool(self.mc)

    def get_metafile(self, path):
        """
        Load the yaml metadata in the given file.
        """
        d = self.metafiles.get(path)
        if d is not None:
            return d

        k = ("PKGYAML", path)
        path_modtime = os.path.getmtime(path)

        if self.mc:
            t = self.mc.get(k)
            if t is not None:
                mtime,d = t
                if path_modtime == mtime:
                    self.metafiles[path] = d
                    return d

        d = rez_metafile.ConfigMetadata(path)
        d.delete_nonessentials()

        if self.mc:
            self.mc.set(k, (path_modtime, d))
        self.metafiles[path] = d
        return d

    def get_versions_in_directory(self, path, warnings=True):
        """
        For a given directory, return a list of (Version,epoch), which match version directories 
        found in the given directory.
        """
        vers = self.versions.get(path)
        if vers is not None:
            return vers

        if not os.path.isdir(path):
            return []

        k = ("VERSIONS", path)
        path_modtime = os.path.getmtime(path)

        if self.mc:
            t = self.mc.get(k)
            if t is not None:
                mtime,tvers = t
                if path_modtime == mtime:
                    vers = [x for x in tvers if x[1] <= self.epoch]
                    self.versions[path] = vers
                    return vers

        tvers = rez_filesys.get_versions_in_directory(path, warnings)
        if self.mc:
            self.mc.set(k, (path_modtime, tvers))
        vers = [x for x in tvers if x[1] <= self.epoch]
        self.versions[path] = vers
        return vers

    def iter_packages(self, family_name, paths=None):
        """
        Given a family name and a `VersionRange`, iterate through
        (family path, resolved `Version`, epoch) for all versions found.
        """
        if paths is None:
            paths = rez_filesys._g_syspaths

        for pkg_path in paths:
            family_path = os.path.join(pkg_path, family_name)
            vers = self.get_versions_in_directory(family_path)
            if vers:
                for ver, timestamp in vers:
                    yield family_path, ver, timestamp
            elif os.path.isfile(os.path.join(family_path, PKG_METADATA_FILENAME)):
                # check for special case - unversioned package.
                # only allowed when no versioned packages exist.
                yield family_path, Version(""), 0

    def find_package_in_range(self, family_name, ver_range, latest=True, exact=False,
                    paths=None):
        """<commit_msg>rez_memcached: hide some of the inner workings of the memcache from the functions being cached by adding a cached_path decorator that does all of the work.<commit_after>import sys
import os
import time
from collections import defaultdict
import rez_filesys
import rez_metafile
from versions import *
from public_enums import *
from rez_exceptions import *



_g_caching_enabled = True
_g_memcached_server = os.getenv("REZ_MEMCACHED_SERVER") or "127.0.0.1:11211"


def _create_client():
    if not _g_caching_enabled:
        return None
    return memcache.Client([_g_memcached_server], cache_cas=True)


def print_cache_warning(msg):
    print >> sys.stderr, "Cache Warning: %s" % msg

def _filter_epoch(vers, cache):
    return [x for x in vers if x[1] <= cache.epoch]

# init
_g_caching_enabled = not os.getenv("REZ_DISABLE_CACHING")
if _g_caching_enabled:
    try:
        import memcache
        from memcached_client import *
    except:
        _g_caching_enabled = False
if _g_caching_enabled:
    mc = _create_client()
    if not mc.set("test_set", "success"):
        _g_caching_enabled = False
    mc = None

def cached_path(key, default=None, postfilter=None):
    """
    A decorator to aid in automatically caching functions

    key : unique key used to identify the data to cache with the memcache client
    default : value to return if the path does not exist
    postfilter : filter function to apply to after retrieving data from the memcache client.
        should take the data and an instance of the memcache as arguments and
        return a modified copy of data.
    """
    def decorator(func):
        def wrapped_func(self, path, *args, **kwargs):
            # get the cache for this key.
            # self.cache is a defaultdict, so this will always return a dict
            cache = self.cache[key]

            # FIXME: allow None to be cached.
            data = cache.get(path)
            if data is not None:
                return data
 
            try:
                path_modtime = os.path.getmtime(path)
            except OSError:
                if default is not None:
                    return default
                raise

            k = (key, path)
            # get memcached data if it exists
            if self.mc:
                # FIXME: allow None to be cached.
                t = self.mc.get(k)
                if t is not None:
                    mtime, d = t
                    if path_modtime == mtime:
                        data = d

            if not data:
                # get data
                data = func(self, path, *args, **kwargs)

            assert data is not None, "Cached function must not return None"

            # cache result to memcache
            if self.mc:
                self.mc.set(k, (path_modtime, data))

            if postfilter:
                data = postfilter(data, self)
            
            # cache result to local instance cache
            cache[path] = data
            return data

        wrapped_func.__name__ = func.__name__
        wrapped_func.__doc__ = func.__doc__
        wrapped_func.__module__ = func.__module__
        return wrapped_func
    return decorator

class RezMemCache(object):
    """
    Cache for filesystem access and resolves.
    """
    def __init__(self, time_epoch=0, use_caching=True):
        self.epoch = time_epoch or int(time.time())
        self.cache = defaultdict(dict)
        self.families = set()
        self.mc = None
        if use_caching and _g_caching_enabled:
            mc = _create_client()
            self.mc = MemCacheClient(mc)

    def caching_enabled(self):
        """
        whether the memcache client is being used
        """
        return bool(self.mc)

    @cached_path("PKGYAML")
    def get_metafile(self, path):
        """
        Load the yaml metadata in the given file.
        """
        d = rez_metafile.ConfigMetadata(path)
        d.delete_nonessentials()
        return d

    @cached_path("VERSIONS", default=(), postfilter=_filter_epoch)
    def get_versions_in_directory(self, path, warnings=True):
        """
        For a given directory, return a list of (Version,epoch), which match version directories 
        found in the given directory.
        """
        return rez_filesys.get_versions_in_directory(path, warnings)

    def _find_package(self, path, ver_range, latest=True, exact=False):
        """
        Given a path to a package family and a VersionRange, return (resolved
        Version, epoch) or None if not found.
        """
        vers = self.get_versions_in_directory(path)

        # check for special case - unversioned package
        # todo subtle bug here, unversioned pkg's timestamp not taken into account. In practice
        # though this should not cause any problems.
        if not vers and ver_range.is_any() and os.path.isfile(os.path.join(path, PKG_METADATA_FILENAME)):
            return (Version(""), 0)

        if not ver_range.is_inexact():
            exact_ver = [x for x in vers if x[0] == ver_range.versions[0]]
            if exact_ver:
                return exact_ver[0]

        if exact:
            return None

        # find the earliest/latest version on disk that falls within ver
        if latest:
            vers = reversed(vers)
        for ver in vers:
            if ver_range.contains_version(ver[0]):
                return ver

        return None

    def iter_packages(self, family_name, paths=None):
        """
        Given a family name and a `VersionRange`, iterate through
        (family path, resolved `Version`, epoch) for all versions found.
        """
        if paths is None:
            paths = rez_filesys._g_syspaths

        for pkg_path in paths:
            family_path = os.path.join(pkg_path, family_name)
            vers = self.get_versions_in_directory(family_path)
            if vers:
                for ver, timestamp in vers:
                    yield family_path, ver, timestamp
            elif os.path.isfile(os.path.join(family_path, PKG_METADATA_FILENAME)):
                # check for special case - unversioned package.
                # only allowed when no versioned packages exist.
                yield family_path, Version(""), 0

    def find_package_in_range(self, family_name, ver_range, latest=True, exact=False,
                    paths=None):
        """
{'fraction_non_alphanumeric': 0.08024750118990957, 'fraction_numerical': 0.003236554021894336, 'mean_word_length': 1.9804255319148936, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5660738', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2970, 'n_words': 922}