<commit_before>#!/usr/bin/env python
# coding: utf-8

# gitshelve.py, version 0.1
#
# by <NAME> <<EMAIL>>
#
# This file implements a Python shelve object that uses a branch within the
# current Git repository to store its data, plus a history of all changes to
# that data.  The usage is identical to shelve, with the exception that a
# repository directory or branch name must be specified, and that
# writeback=True is assumed.
#
# Example:
#
#   import gitshelve
#
#   data = gitshelve.open(branch = 'mydata', repository = '/tmp/foo')
#   data = gitshelve.open(branch = 'mydata')  # use current repo
#
#   data['foo/bar/git.c'] = "This is some sample data."
#
#   data.commit("Changes")
#   data.sync()                  # same as data.commit()
#
#   print data['foo/bar/git.c']
#
#   data.close()
#
# If you checkout the 'mydata' branch now, you'll see the file 'git.c' in the
# directory 'foo/bar'.  Running 'git log' will show the change you made.

import re
import os

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO

from subprocess import Popen, PIPE
from string import split, join

######################################################################

verbose = False

######################################################################

# Utility function for calling out to Git (this script does not try to
# be a Git library, just an interface to the underlying commands).  It
# supports a 'restart' keyword, which will cause a Python function to
# be called on failure.  If that function returns True, the same
# command will be attempted again.  This can avoid costly checks to
# make sure a branch exists, for example, by simply failing on the
# first attempt to use it and then allowing the restart function to
# create it.

class GitError(Exception):
    def __init__(self, cmd, args, kwargs, stderr = None):
        self.cmd = cmd
        self.args = args
        self.kwargs = kwargs
        self.stderr = stderr
        Exception.__init__(self)

    def __str__(self):
        if self.stderr:
            return "Git command failed: git %s %s: %s" % \
                (self.cmd, self.args, self.stderr)
        else:
            return "Git command failed: git %s %s" % (self.cmd, self.args)

def git(cmd, *args, **kwargs):
    restart = True
    while restart:
        stdin_mode = None
        if kwargs.has_key('input'):
            stdin_mode = PIPE

        if verbose:
            print "Command: git %s %s" % (cmd, join(args, ' '))
            if kwargs.has_key('input'):
                print "Input: <<EOF"
                print kwargs['input'],
                print "EOF"

        environ = None
        if kwargs.has_key('repository'):
            environ = os.environ.copy()
            environ['GIT_DIR'] = kwargs['repository']

            git_dir = environ['GIT_DIR']
            if not os.path.isdir(git_dir):
                proc = Popen(('git', 'init'), env = environ,
                             stdout = PIPE, stderr = PIPE)
                if proc.wait() != 0:
                    raise GitError('init', [], {}, proc.stderr.read())

        proc = Popen(('git', cmd) + args, env = environ,
                     stdin  = stdin_mode,
                     stdout = PIPE,
                     stderr = PIPE)

        if kwargs.has_key('input'):
            proc.stdin.write(kwargs['input'])
            proc.stdin.close()

        returncode = proc.wait()
        restart = False
        if returncode != 0:
            if kwargs.has_key('restart'):
                if kwargs['restart'](cmd, args, kwargs):
                    restart = True
            else:
                raise GitError(cmd, args, kwargs, proc.stderr.read())

    if not kwargs.has_key('ignore_output'):
        if kwargs.has_key('keep_newline'):
            return proc.stdout.read()
        else:
            return proc.stdout.read()[:-1]


class gitbook:<commit_msg>add working tree support to git()<commit_after>#!/usr/bin/env python
# coding: utf-8

# gitshelve.py, version 0.1
#
# by <NAME> <<EMAIL>>
#
# This file implements a Python shelve object that uses a branch within the
# current Git repository to store its data, plus a history of all changes to
# that data.  The usage is identical to shelve, with the exception that a
# repository directory or branch name must be specified, and that
# writeback=True is assumed.
#
# Example:
#
#   import gitshelve
#
#   data = gitshelve.open(branch = 'mydata', repository = '/tmp/foo')
#   data = gitshelve.open(branch = 'mydata')  # use current repo
#
#   data['foo/bar/git.c'] = "This is some sample data."
#
#   data.commit("Changes")
#   data.sync()                  # same as data.commit()
#
#   print data['foo/bar/git.c']
#
#   data.close()
#
# If you checkout the 'mydata' branch now, you'll see the file 'git.c' in the
# directory 'foo/bar'.  Running 'git log' will show the change you made.

import re
import os

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO

from subprocess import Popen, PIPE
from string import split, join

######################################################################

verbose = False

######################################################################

# Utility function for calling out to Git (this script does not try to
# be a Git library, just an interface to the underlying commands).  It
# supports a 'restart' keyword, which will cause a Python function to
# be called on failure.  If that function returns True, the same
# command will be attempted again.  This can avoid costly checks to
# make sure a branch exists, for example, by simply failing on the
# first attempt to use it and then allowing the restart function to
# create it.

class GitError(Exception):
    def __init__(self, cmd, args, kwargs, stderr = None):
        self.cmd = cmd
        self.args = args
        self.kwargs = kwargs
        self.stderr = stderr
        Exception.__init__(self)

    def __str__(self):
        if self.stderr:
            return "Git command failed: git %s %s: %s" % \
                (self.cmd, self.args, self.stderr)
        else:
            return "Git command failed: git %s %s" % (self.cmd, self.args)

def git(cmd, *args, **kwargs):
    restart = True
    while restart:
        stdin_mode = None
        if kwargs.has_key('input'):
            stdin_mode = PIPE

        if verbose:
            print "Command: git %s %s" % (cmd, join(args, ' '))
            if kwargs.has_key('input'):
                print "Input: <<EOF"
                print kwargs['input'],
                print "EOF"

        environ = None
        if kwargs.has_key('repository'):
            environ = os.environ.copy()
            environ['GIT_DIR'] = kwargs['repository']

            git_dir = environ['GIT_DIR']
            if not os.path.isdir(git_dir):
                proc = Popen(('git', 'init'), env = environ,
                             stdout = PIPE, stderr = PIPE)
                if proc.wait() != 0:
                    raise GitError('init', [], {}, proc.stderr.read())

        if 'worktree' in kwargs:
            if environ is None:
                environ = os.environ.copy()
            environ['GIT_WORK_TREE'] = kwargs['worktree']
            work_tree = environ['GIT_WORK_TREE']
            if not os.path.isdir(work_tree):
                os.makedirs(work_tree)

        proc = Popen(('git', cmd) + args, env = environ,
                     stdin  = stdin_mode,
                     stdout = PIPE,
                     stderr = PIPE)

        if kwargs.has_key('input'):
            proc.stdin.write(kwargs['input'])
            proc.stdin.close()

        returncode = proc.wait()
        restart = False
        if returncode != 0:
            if kwargs.has_key('restart'):
                if kwargs['restart'](cmd, args, kwargs):
                    restart = True
            else:
                raise GitError(cmd, args, kwargs, proc.stderr.read())

    if not kwargs.has_key('ignore_output'):
        if kwargs.has_key('keep_newline'):
            return proc.stdout.read()
        else:
            return proc.stdout.read()[:-1]


class gitbook:
{'fraction_non_alphanumeric': 0.14082608162765045, 'fraction_numerical': 0.0014707684765289863, 'mean_word_length': 2.213863725876329, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4956785', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2319, 'n_words': 778}