<commit_before>from os import remove, devnull
from hashlib import md5, sha1
from cutools.vcs import VCSInterface
from cutools.diff import get_chunks
from plumbum.cmd import git
from plumbum.commands import ProcessExecutionError

class Git(VCSInterface):
    """Git implementation for Check Upgrade Tools.
    """
    def __init__(self, upstream):
        self.upstream = upstream

    def get_md5_files(self):
        res = []
        files = [x for x in git['ls-files', '-mo',
                                '--exclude-standard']().split('\n') if x]
        for fl in files:
            cmd = git['show', '%s:%s' % (self.upstream, fl)]
            try:
                pymd5 = md5(unicode(cmd()).encode('utf-8')).hexdigest()
                res += [(pymd5, fl)]
            except ProcessExecutionError:
                res += [(' ' * 32, fl)]
                continue
        return res

    def fetch(self, remote=None):
        if not remote:
            remote = self.upstream.split('/')[0]
        git['fetch', remote]()

    def checkout(self, check_file):
        git['checkout', '--', check_file]()

    def merge(self, upstream=None):
        if not upstream:
            upstream = self.upstream
        if '/' in upstream:
            self.fetch(upstream.split('/')[0])
        git['reset', '--hard', self.local_rev]()
        git['merge', upstream]()

    def apply_diff(self, diff):<commit_msg>FIX getting files in git1.5<commit_after>from os import remove, devnull
from hashlib import md5, sha1
from cutools.vcs import VCSInterface
from cutools.diff import get_chunks
from plumbum.cmd import git
from plumbum.commands import ProcessExecutionError

class Git(VCSInterface):
    """Git implementation for Check Upgrade Tools.
    """
    def __init__(self, upstream):
        self.upstream = upstream

    def get_md5_files(self):
        res = []
        files = [x for x in git['ls-files', '-m', '-o',
                                '--exclude-standard']().split('\n') if x]
        for fl in files:
            cmd = git['show', '%s:%s' % (self.upstream, fl)]
            try:
                pymd5 = md5(unicode(cmd()).encode('utf-8')).hexdigest()
                res += [(pymd5, fl)]
            except ProcessExecutionError:
                res += [(' ' * 32, fl)]
                continue
        return res

    def fetch(self, remote=None):
        if not remote:
            remote = self.upstream.split('/')[0]
        git['fetch', remote]()

    def checkout(self, check_file):
        git['checkout', '--', check_file]()

    def merge(self, upstream=None):
        if not upstream:
            upstream = self.upstream
        if '/' in upstream:
            self.fetch(upstream.split('/')[0])
        git['reset', '--hard', self.local_rev]()
        git['merge', upstream]()

    def apply_diff(self, diff):
{'fraction_non_alphanumeric': 0.13923158265773705, 'fraction_numerical': 0.008459640465280225, 'mean_word_length': 2.0714285714285716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '187218', 'n_tokens_mistral': 873, 'n_tokens_neox': 830, 'n_words': 224}