<commit_before>import os
import os.path
import sys
import subprocess
import re
import tempfile
from util import *

FILE_ADDED       = 'A'
FILE_MODIFIED    = 'M'
FILE_DELETED     = 'D'
FILE_COPIED      = 'C'
FILE_RENAMED     = 'R'
FILE_UNMERGED    = 'U'
FILE_TYPECHANGED = 'T'
FILE_UNTRACKED   = 'N'
FILE_BROKEN      = 'B'
FILE_UNKNOWN     = 'X'

MERGE_TOOLS = {
    'diffmerge.app': (
        ['/Applications/DiffMerge.app/Contents/MacOS/DiffMerge'],
        ['--nosplash', '-t1={FILENAME}.LOCAL', '-t2={FILENAME}.MERGED', '-t3={FILENAME}.REMOTE', '{LOCAL}', '{MERGED}', '{REMOTE}']
    ),
    'diffmerge.cmdline': (
        ['{PATH}/diffmerge', '{PATH}/diffmerge.sh'],
        ['--nosplash', '-t1=LOCAL', '-t2=MERGED', '-t3=REMOTE', '{LOCAL}', '{MERGED}', '{REMOTE}']
    ),
    'meld': (
        ['{PATH}/meld'],
        ['{LOCAL}', '{MERGED}', '{REMOTE}']
    ),
    'winmerge.win32': (
        [r'C:\Program Files\WinMerge\WinMergeU.exe'],
        ['{MERGED}'] # It does not support 3-way merge yet...
    ),
    'winmerge.cygwin': (
        ['/c/Program Files/WinMerge/WinMergeU.exe'],
        ['{MERGED}'] # It does not support 3-way merge yet...
    )
}

_git = None
commit_pool = {}

class GitError(RuntimeError): pass

def git_binary():
    global _git

    if _git:
        return _git

    # Search for git binary
    if os.name == 'posix':
        locations = ['{PATH}/git', '/opt/local/bin/git', '/usr/local/git/bin']
    elif sys.platform == 'win32':
        locations = (r'{PATH}\git.exe', r'C:\Program Files\Git\bin\git.exe')
    elif sys.platform == 'cygwin':
        locations = (r'{PATH}/git.exe', r'/c/Program Files/Git/bin/git.exe')
    else:
        locations = []

    for _git in find_binary(locations):
        return _git

    _git = None
    raise GitError, "git executable not found"

_mergetool = None
def detect_mergetool():
    global _mergetool

    if _mergetool:
        return _mergetool

    # Select tools
    if sys.platform == 'darwin':
        # Mac OS X
        tools = ['diffmerge.app', 'diffmerge.cmdline', 'meld']
    elif os.name == 'posix':
        # Other Unix
        tools = ['diffmerge.cmdline', 'meld']
    elif sys.platform == 'win32':
        # Windows
        tools = ['winmerge.win32']
    elif sys.platform == 'cygwin':
        # Cygwin
        tools = ['winmerge.cygwin']
    else:
        raise GitError, "Cannot detect any merge tool"

    # Detect binaries
    for tool in tools:
        locations, args = MERGE_TOOLS[tool]
        for location in find_binary(locations):
            _mergetool = (location, args)
            return _mergetool

    # Return error if no tool was found
    raise GitError, "Cannot detect any merge tool"

def run_cmd(dir, args, with_retcode=False, with_stderr=False, raise_error=False, input=None, env={}):
    # Check args
    if type(args) in [str, unicode]:
        args = [args]

    # Check directory
    if not os.path.isdir(dir):
        raise GitError, 'Directory not exists: ' + dir

    try:
        os.chdir(dir)
    except OSError, msg:
        raise GitError, msg

    # Run command
    if type(args) != list:
        args = [args]

    git_env = dict(os.environ)
    git_env.update(env)

    p = Popen([git_binary()] + args, stdout=subprocess.PIPE,
              stderr=subprocess.PIPE, stdin=subprocess.PIPE,
              env=git_env, shell=False)

    if input == None:
        stdout,stderr = p.communicate('')
    else:
        stdout,stderr = p.communicate(utf8_str(input))
    
    # Return command output in a form given by arguments
    ret = []

    if p.returncode != 0 and raise_error:
        raise GitError, 'git returned with the following error:\n%s' % stderr

    if with_retcode:
        ret.append(p.returncode)

    ret.append(stdout)

    if with_stderr:
        ret.append(stderr)

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)

class Repository(object):
    def __init__(self, repodir, name='Main module', parent=None):
        self.name = name
        self.parent = parent

        # Search for .git directory in repodir ancestors
        repodir = os.path.abspath(repodir)
        try:
            while not os.path.isdir(os.path.join(repodir, '.git')):
                new_repodir = os.path.abspath(os.path.join(repodir, '..'))
                if new_repodir == repodir:
                    raise GitError, "Directory is not a git repository"
                else:
                    repodir = new_repodir
        except OSError:
            raise GitError, "Directory is not a git repository or it is not readable"
            
        self.dir = repodir

        # Origin URL
        self.config = ConfigFile(os.path.join(self.dir, '.git', 'config'))
        self.url = None
        self.url = self.config.get_option('remote', 'origin', 'url')

        # Run a git status to see whether this is really a git repository
        retcode,output = self.run_cmd(['status'], with_retcode=True)
        if retcode not in [0,1]:
            raise GitError, "Directory is not a git repository"

        # Load refs
        self.load_refs()<commit_msg>prevent infinite loop when a submbodule is missing<commit_after>import os
import os.path
import sys
import subprocess
import re
import tempfile
from util import *

FILE_ADDED       = 'A'
FILE_MODIFIED    = 'M'
FILE_DELETED     = 'D'
FILE_COPIED      = 'C'
FILE_RENAMED     = 'R'
FILE_UNMERGED    = 'U'
FILE_TYPECHANGED = 'T'
FILE_UNTRACKED   = 'N'
FILE_BROKEN      = 'B'
FILE_UNKNOWN     = 'X'

MERGE_TOOLS = {
    'diffmerge.app': (
        ['/Applications/DiffMerge.app/Contents/MacOS/DiffMerge'],
        ['--nosplash', '-t1={FILENAME}.LOCAL', '-t2={FILENAME}.MERGED', '-t3={FILENAME}.REMOTE', '{LOCAL}', '{MERGED}', '{REMOTE}']
    ),
    'diffmerge.cmdline': (
        ['{PATH}/diffmerge', '{PATH}/diffmerge.sh'],
        ['--nosplash', '-t1=LOCAL', '-t2=MERGED', '-t3=REMOTE', '{LOCAL}', '{MERGED}', '{REMOTE}']
    ),
    'meld': (
        ['{PATH}/meld'],
        ['{LOCAL}', '{MERGED}', '{REMOTE}']
    ),
    'winmerge.win32': (
        [r'C:\Program Files\WinMerge\WinMergeU.exe'],
        ['{MERGED}'] # It does not support 3-way merge yet...
    ),
    'winmerge.cygwin': (
        ['/c/Program Files/WinMerge/WinMergeU.exe'],
        ['{MERGED}'] # It does not support 3-way merge yet...
    )
}

_git = None
commit_pool = {}

class GitError(RuntimeError): pass

def git_binary():
    global _git

    if _git:
        return _git

    # Search for git binary
    if os.name == 'posix':
        locations = ['{PATH}/git', '/opt/local/bin/git', '/usr/local/git/bin']
    elif sys.platform == 'win32':
        locations = (r'{PATH}\git.exe', r'C:\Program Files\Git\bin\git.exe')
    elif sys.platform == 'cygwin':
        locations = (r'{PATH}/git.exe', r'/c/Program Files/Git/bin/git.exe')
    else:
        locations = []

    for _git in find_binary(locations):
        return _git

    _git = None
    raise GitError, "git executable not found"

_mergetool = None
def detect_mergetool():
    global _mergetool

    if _mergetool:
        return _mergetool

    # Select tools
    if sys.platform == 'darwin':
        # Mac OS X
        tools = ['diffmerge.app', 'diffmerge.cmdline', 'meld']
    elif os.name == 'posix':
        # Other Unix
        tools = ['diffmerge.cmdline', 'meld']
    elif sys.platform == 'win32':
        # Windows
        tools = ['winmerge.win32']
    elif sys.platform == 'cygwin':
        # Cygwin
        tools = ['winmerge.cygwin']
    else:
        raise GitError, "Cannot detect any merge tool"

    # Detect binaries
    for tool in tools:
        locations, args = MERGE_TOOLS[tool]
        for location in find_binary(locations):
            _mergetool = (location, args)
            return _mergetool

    # Return error if no tool was found
    raise GitError, "Cannot detect any merge tool"

def run_cmd(dir, args, with_retcode=False, with_stderr=False, raise_error=False, input=None, env={}):
    # Check args
    if type(args) in [str, unicode]:
        args = [args]

    # Check directory
    if not os.path.isdir(dir):
        raise GitError, 'Directory not exists: ' + dir

    try:
        os.chdir(dir)
    except OSError, msg:
        raise GitError, msg

    # Run command
    if type(args) != list:
        args = [args]

    git_env = dict(os.environ)
    git_env.update(env)

    p = Popen([git_binary()] + args, stdout=subprocess.PIPE,
              stderr=subprocess.PIPE, stdin=subprocess.PIPE,
              env=git_env, shell=False)

    if input == None:
        stdout,stderr = p.communicate('')
    else:
        stdout,stderr = p.communicate(utf8_str(input))
    
    # Return command output in a form given by arguments
    ret = []

    if p.returncode != 0 and raise_error:
        raise GitError, 'git returned with the following error:\n%s' % stderr

    if with_retcode:
        ret.append(p.returncode)

    ret.append(stdout)

    if with_stderr:
        ret.append(stderr)

    if len(ret) == 1:
        return ret[0]
    else:
        return tuple(ret)

class Repository(object):
    def __init__(self, repodir, name='Main module', parent=None):
        self.name = name
        self.parent = parent

        # Search for .git directory in repodir ancestors
        repodir = os.path.abspath(repodir)
        try:
            while not os.path.isdir(os.path.join(repodir, '.git')):
                new_repodir = os.path.abspath(os.path.join(repodir, '..'))
                if new_repodir == repodir or new_repodir == parent.dir:
                    raise GitError, "Directory is not a git repository"
                else:
                    repodir = new_repodir
        except OSError:
            raise GitError, "Directory is not a git repository or it is not readable"
            
        self.dir = repodir

        # Origin URL
        self.config = ConfigFile(os.path.join(self.dir, '.git', 'config'))
        self.url = None
        self.url = self.config.get_option('remote', 'origin', 'url')

        # Run a git status to see whether this is really a git repository
        retcode,output = self.run_cmd(['status'], with_retcode=True)
        if retcode not in [0,1]:
            raise GitError, "Directory is not a git repository"

        # Load refs
        self.load_refs()
{'fraction_non_alphanumeric': 0.1344284603421462, 'fraction_numerical': 0.0042768273716951785, 'mean_word_length': 2.2736239261851736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3786938', 'n_tokens_mistral': 3515, 'n_tokens_neox': 3360, 'n_words': 859}