<commit_before># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2006-2011, <NAME> <<EMAIL>>
#
# See COPYING for distribution information

from __future__ import with_statement

from future27 import namedtuple

import os, re, sys, time, weakref
from collections import deque
from functools import partial
from threading import Lock
from subprocess import Popen, PIPE
from operator import itemgetter
from contextlib import contextmanager
import cStringIO
import codecs

__all__ = ["git_version", "GitError", "GitErrorSha", "Storage", "StorageFactory"]

class GitError(Exception):
    pass

class GitErrorSha(GitError):
    pass

class GitCore(object):
    """
    Low-level wrapper around git executable
    """

    def __init__(self, git_dir=None, git_bin="git"):
        self.__git_bin = git_bin
        self.__git_dir = git_dir

    def __repr__(self):
        return '<GitCore bin="%s" dir="%s">' % (self.__git_bin, self.__git_dir)

    def __build_git_cmd(self, gitcmd, *args):
        "construct command tuple for git call suitable for Popen()"

        cmd = [self.__git_bin]
        if self.__git_dir:
            cmd.append('--git-dir=%s' % self.__git_dir)
        cmd.append(gitcmd)
        cmd.extend(args)

        return cmd

    def __pipe(self, git_cmd, *cmd_args, **kw):
        if sys.platform == "win32":
            return Popen(self.__build_git_cmd(git_cmd, *cmd_args), **kw)
        else:
            return Popen(self.__build_git_cmd(git_cmd, *cmd_args),
                         close_fds=True, **kw)

    def __execute(self, git_cmd, *cmd_args):
        "execute git command and return file-like object of stdout"

        #print >>sys.stderr, "DEBUG:", git_cmd, cmd_args

        p = self.__pipe(git_cmd, *cmd_args, stdout=PIPE, stderr=PIPE)

        stdout_data, stderr_data = p.communicate()
        #TODO, do something with p.returncode, e.g. raise exception

        return stdout_data

    def cat_file_batch(self):
        return self.__pipe('cat-file', '--batch', stdin=PIPE, stdout=PIPE)

    def log_pipe(self, *cmd_args):
        return self.__pipe('log', *cmd_args, stdout=PIPE)

    def __getattr__(self, name):
        if name[0] == '_' or name in ['cat_file_batch', 'log_pipe']:
            raise AttributeError, name
        return partial(self.__execute, name.replace('_','-'))

    __is_sha_pat = re.compile(r'[0-9A-Fa-f]*$')

    @classmethod
    def is_sha(cls, sha):
        """
        returns whether sha is a potential sha id
        (i.e. proper hexstring between 4 and 40 characters)
        """

        # quick test before starting up regexp matcher
        if not (4 <= len(sha) <= 40):
            return False

        return bool(cls.__is_sha_pat.match(sha))

class SizedDict(dict):
    """
    Size-bounded dictionary with FIFO replacement strategy
    """

    def __init__(self, max_size=0):
        dict.__init__(self)
        self.__max_size = max_size
        self.__key_fifo = deque()
        self.__lock = Lock()

    def __setitem__(self, name, value):
        with self.__lock:
            assert len(self) == len(self.__key_fifo) # invariant

            if not self.__contains__(name):
                self.__key_fifo.append(name)

            rc = dict.__setitem__(self, name, value)

            while len(self.__key_fifo) > self.__max_size:
                self.__delitem__(self.__key_fifo.popleft())

            assert len(self) == len(self.__key_fifo) # invariant

            return rc

    def setdefault(self, *_):
        raise NotImplemented("SizedDict has no setdefault() method")

class StorageFactory(object):
    __dict = weakref.WeakValueDictionary()
    __dict_nonweak = dict()
    __dict_lock = Lock()

    def __init__(self, repo, log, weak=True, git_bin='git', git_fs_encoding=None):
        self.logger = log

        with StorageFactory.__dict_lock:
            try:
                i = StorageFactory.__dict[repo]
            except KeyError:
                i = Storage(repo, log, git_bin, git_fs_encoding)
                StorageFactory.__dict[repo] = i

                # create or remove additional reference depending on 'weak' argument
                if weak:
                    try:
                        del StorageFactory.__dict_nonweak[repo]
                    except KeyError:
                        pass
                else:
                    StorageFactory.__dict_nonweak[repo] = i

        self.__inst = i
        self.__repo = repo

    def getInstance(self):
        is_weak = self.__repo not in StorageFactory.__dict_nonweak
        self.logger.debug("requested %sPyGIT.Storage instance %d for '%s'"
                          % (("","weak ")[is_weak], id(self.__inst), self.__repo))
        return self.__inst


class Storage(object):
    """
    High-level wrapper around GitCore with in-memory caching
    """

    __SREV_MIN = 4 # minimum short-rev length

    RevCache = namedtuple('RevCache', 'youngest_rev oldest_rev rev_dict tag_set srev_dict branch_dict')

    @staticmethod
    def __rev_key(rev):
        assert len(rev) >= 4
        #assert GitCore.is_sha(rev)
        srev_key = int(rev[:4], 16)
        assert srev_key >= 0 and srev_key <= 0xffff
        return srev_key

    @staticmethod
    def git_version(git_bin="git"):
        GIT_VERSION_MIN_REQUIRED = (1, 5, 6)<commit_msg>TracGit: Replace usage of namedtuple (Python 2.7) with result of that code generator to define `Storage.RevCache`.<commit_after># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2006-2011, <NAME> <<EMAIL>>
#
# See COPYING for distribution information

from __future__ import with_statement

import os, re, sys, time, weakref
from collections import deque
from functools import partial
from threading import Lock
from subprocess import Popen, PIPE
from operator import itemgetter
from contextlib import contextmanager
import cStringIO
import codecs

__all__ = ["git_version", "GitError", "GitErrorSha", "Storage", "StorageFactory"]

class GitError(Exception):
    pass

class GitErrorSha(GitError):
    pass

class GitCore(object):
    """
    Low-level wrapper around git executable
    """

    def __init__(self, git_dir=None, git_bin="git"):
        self.__git_bin = git_bin
        self.__git_dir = git_dir

    def __repr__(self):
        return '<GitCore bin="%s" dir="%s">' % (self.__git_bin, self.__git_dir)

    def __build_git_cmd(self, gitcmd, *args):
        "construct command tuple for git call suitable for Popen()"

        cmd = [self.__git_bin]
        if self.__git_dir:
            cmd.append('--git-dir=%s' % self.__git_dir)
        cmd.append(gitcmd)
        cmd.extend(args)

        return cmd

    def __pipe(self, git_cmd, *cmd_args, **kw):
        if sys.platform == "win32":
            return Popen(self.__build_git_cmd(git_cmd, *cmd_args), **kw)
        else:
            return Popen(self.__build_git_cmd(git_cmd, *cmd_args),
                         close_fds=True, **kw)

    def __execute(self, git_cmd, *cmd_args):
        "execute git command and return file-like object of stdout"

        #print >>sys.stderr, "DEBUG:", git_cmd, cmd_args

        p = self.__pipe(git_cmd, *cmd_args, stdout=PIPE, stderr=PIPE)

        stdout_data, stderr_data = p.communicate()
        #TODO, do something with p.returncode, e.g. raise exception

        return stdout_data

    def cat_file_batch(self):
        return self.__pipe('cat-file', '--batch', stdin=PIPE, stdout=PIPE)

    def log_pipe(self, *cmd_args):
        return self.__pipe('log', *cmd_args, stdout=PIPE)

    def __getattr__(self, name):
        if name[0] == '_' or name in ['cat_file_batch', 'log_pipe']:
            raise AttributeError, name
        return partial(self.__execute, name.replace('_','-'))

    __is_sha_pat = re.compile(r'[0-9A-Fa-f]*$')

    @classmethod
    def is_sha(cls, sha):
        """
        returns whether sha is a potential sha id
        (i.e. proper hexstring between 4 and 40 characters)
        """

        # quick test before starting up regexp matcher
        if not (4 <= len(sha) <= 40):
            return False

        return bool(cls.__is_sha_pat.match(sha))

class SizedDict(dict):
    """
    Size-bounded dictionary with FIFO replacement strategy
    """

    def __init__(self, max_size=0):
        dict.__init__(self)
        self.__max_size = max_size
        self.__key_fifo = deque()
        self.__lock = Lock()

    def __setitem__(self, name, value):
        with self.__lock:
            assert len(self) == len(self.__key_fifo) # invariant

            if not self.__contains__(name):
                self.__key_fifo.append(name)

            rc = dict.__setitem__(self, name, value)

            while len(self.__key_fifo) > self.__max_size:
                self.__delitem__(self.__key_fifo.popleft())

            assert len(self) == len(self.__key_fifo) # invariant

            return rc

    def setdefault(self, *_):
        raise NotImplemented("SizedDict has no setdefault() method")

class StorageFactory(object):
    __dict = weakref.WeakValueDictionary()
    __dict_nonweak = dict()
    __dict_lock = Lock()

    def __init__(self, repo, log, weak=True, git_bin='git', git_fs_encoding=None):
        self.logger = log

        with StorageFactory.__dict_lock:
            try:
                i = StorageFactory.__dict[repo]
            except KeyError:
                i = Storage(repo, log, git_bin, git_fs_encoding)
                StorageFactory.__dict[repo] = i

                # create or remove additional reference depending on 'weak' argument
                if weak:
                    try:
                        del StorageFactory.__dict_nonweak[repo]
                    except KeyError:
                        pass
                else:
                    StorageFactory.__dict_nonweak[repo] = i

        self.__inst = i
        self.__repo = repo

    def getInstance(self):
        is_weak = self.__repo not in StorageFactory.__dict_nonweak
        self.logger.debug("requested %sPyGIT.Storage instance %d for '%s'"
                          % (("","weak ")[is_weak], id(self.__inst), self.__repo))
        return self.__inst


class Storage(object):
    """
    High-level wrapper around GitCore with in-memory caching
    """

    __SREV_MIN = 4 # minimum short-rev length


    class RevCache(tuple):
        """RevCache(youngest_rev, oldest_rev, rev_dict, tag_set, srev_dict, branch_dict)

        In Python 2.7 this class could be defined by:
            from collections import namedtuple
            RevCache = namedtuple('RevCache', 'youngest_rev oldest_rev rev_dict tag_set srev_dict branch_dict')
        This implementation is what that code generator would produce.
        """

        __slots__ = () 

        _fields = ('youngest_rev', 'oldest_rev', 'rev_dict', 'tag_set', 'srev_dict', 'branch_dict') 

        def __new__(cls, youngest_rev, oldest_rev, rev_dict, tag_set, srev_dict, branch_dict):
            return tuple.__new__(cls, (youngest_rev, oldest_rev, rev_dict, tag_set, srev_dict, branch_dict)) 

        @classmethod
        def _make(cls, iterable, new=tuple.__new__, len=len):
            'Make a new RevCache object from a sequence or iterable'
            result = new(cls, iterable)
            if len(result) != 6:
                raise TypeError('Expected 6 arguments, got %d' % len(result))
            return result 

        def __repr__(self):
            return 'RevCache(youngest_rev=%r, oldest_rev=%r, rev_dict=%r, tag_set=%r, srev_dict=%r, branch_dict=%r)' % self 

        def _asdict(t):
            'Return a new dict which maps field names to their values'
            return {'youngest_rev': t[0], 'oldest_rev': t[1], 'rev_dict': t[2], 'tag_set': t[3], 'srev_dict': t[4], 'branch_dict': t[5]} 

        def _replace(self, **kwds):
            'Return a new RevCache object replacing specified fields with new values'
            result = self._make(map(kwds.pop, ('youngest_rev', 'oldest_rev', 'rev_dict', 'tag_set', 'srev_dict', 'branch_dict'), self))
            if kwds:
                raise ValueError('Got unexpected field names: %r' % kwds.keys())
            return result 

        def __getnewargs__(self):
            return tuple(self) 

        youngest_rev = property(itemgetter(0))
        oldest_rev = property(itemgetter(1))
        rev_dict = property(itemgetter(2))
        tag_set = property(itemgetter(3))
        srev_dict = property(itemgetter(4))
        branch_dict = property(itemgetter(5))


    @staticmethod
    def __rev_key(rev):
        assert len(rev) >= 4
        #assert GitCore.is_sha(rev)
        srev_key = int(rev[:4], 16)
        assert srev_key >= 0 and srev_key <= 0xffff
        return srev_key

    @staticmethod
    def git_version(git_bin="git"):
        GIT_VERSION_MIN_REQUIRED = (1, 5, 6)
{'fraction_non_alphanumeric': 0.10812715314751017, 'fraction_numerical': 0.007046664578766051, 'mean_word_length': 2.4317571198280494, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4536382', 'n_tokens_mistral': 4028, 'n_tokens_neox': 3829, 'n_words': 1005}