<commit_before>import os
import re
import shutil
import logging
from binascii import b2a_hex
from datetime import datetime
from cStringIO import StringIO
from ConfigParser import ConfigParser

import tg
os.environ['HGRCPATH'] = '' # disable loading .hgrc
from mercurial import ui, hg
from pymongo.errors import DuplicateKeyError

from ming.base import Object
from ming.orm import Mapper, session
from ming.utils import LazyProperty

from allura import model as M
from allura.lib import helpers as h
from allura.model.repository import topological_sort, GitLikeTree

log = logging.getLogger(__name__)

class Repository(M.Repository):
    tool_name='Hg'
    repo_id='hg'
    type_s='Hg Repository'
    class __mongometa__:
        name='hg-repository'

    @LazyProperty
    def _impl(self):
        return HgImplementation(self)

    def merge_command(self, merge_request):
        '''Return the command to merge a given commit into a given target branch'''
        return 'hg checkout %s\nhg pull -r %s %s' % (
            merge_request.target_branch,
            merge_request.downstream.commit_id,
            merge_request.downstream_repo_url,
        )

    def count(self, branch='default'):
        return super(Repository, self).count(branch)

    def log(self, branch='default', offset=0, limit=10):
        return super(Repository, self).log(branch, offset, limit)

class HgImplementation(M.RepositoryImplementation):
    re_hg_user = re.compile('(.*) <(.*)>')

    def __init__(self, repo):
        self._repo = repo

    @LazyProperty
    def _hg(self):
        return hg.repository(ui.ui(), self._repo.full_fs_path)

    def init(self):
        fullname = self._setup_paths()
        log.info('hg init %s', fullname)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        repo = hg.repository(
            ui.ui(), self._repo.full_fs_path, create=True)
        self.__dict__['_hg'] = repo
        self._setup_special_files()
        self._repo.status = 'ready'

    def clone_from(self, source_url):
        '''Initialize a repo as a clone of another'''
        fullname = self._setup_paths(create_repo_dir=False)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        log.info('Initialize %r as a clone of %s',
                 self._repo, source_url)
        # !$ hg doesn't like unicode as urls
        src, repo = hg.clone(
            ui.ui(),
            source_url.encode('utf-8'),
            self._repo.full_fs_path.encode('utf-8'),
            update=False)
        self.__dict__['_hg'] = repo
        self._setup_special_files()
        self._repo.status = 'analyzing'
        session(self._repo).flush()
        log.info('... %r cloned, analyzing', self._repo)
        self._repo.refresh(notify=False)
        self._repo.status = 'ready'
        log.info('... %s ready', self._repo)
        session(self._repo).flush()

    def commit(self, rev):
        '''Return a Commit object.  rev can be _id or a branch/tag name'''
        # See if the rev is a named ref that we have cached, and use the sha1
        # from the cache. This ensures that we don't return a sha1 that we
        # don't have indexed into mongo yet.
        for ref in self._repo.heads + self._repo.branches + self._repo.repo_tags:
            if ref.name == rev:
                rev = ref.object_id
                break
        result = M.repo.Commit.query.get(_id=rev)
        if result is None:
            try:
                impl = self._hg[str(rev)]
                result = M.repo.Commit.query.get(_id=impl.hex())
            except Exception, e:
                log.exception(e)
        if result is None: return None
        result.set_context(self._repo)
        return result

    def real_parents(self, ci):
        """Return all parents of a commit, excluding the 'null revision' (a
        fake revision added as the parent of the root commit by the Hg api).
        """
        return [p for p in ci.parents() if p]

    def all_commit_ids(self):
        """Return a list of commit ids, starting with the root (first commit)
        of the tree and ending with the head(s).

        NB: The ForgeGit implementation returns commits in the opposite order.
        """
        graph = {}
        to_visit = [ self._hg[hd] for hd in self._hg.heads() ]
        while to_visit:
            obj = to_visit.pop()
            if obj.hex() in graph: continue
            parents = self.real_parents(obj)
            graph[obj.hex()] = set(
                p.hex() for p in parents
                if p.hex() != obj.hex())
            to_visit += parents
        return [ ci for ci in topological_sort(graph) ]

    def new_commits(self, all_commits=False):
        graph = {}
        to_visit = [ self._hg[hd] for hd in self._hg.heads() ]
        while to_visit:
            obj = to_visit.pop()
            if obj.hex() in graph: continue
            if not all_commits:
                # Look up the object
                if M.repo.Commit.query.find(dict(_id=obj.hex())).count():
                    graph[obj.hex()] = set() # mark as parentless
                    continue
            parents = self.real_parents(obj)
            graph[obj.hex()] = set(
                p.hex() for p in parents
                if p.hex() != obj.hex())
            to_visit += parents
        return list(topological_sort(graph))

    def refresh_heads(self):
        self._repo.heads = [
            Object(name=None, object_id=self._hg[head].hex())
            for head in self._hg.heads() ]
        self._repo.branches = [
            Object(name=name, object_id=self._hg[tag].hex())
            for name, tag in self._hg.branchtags().iteritems() ]
        self._repo.repo_tags = [
            Object(name=name, object_id=self._hg[tag].hex())
            for name, tag in self._hg.tags().iteritems() ]
        session(self._repo).flush()

    def refresh_commit_info(self, oid, seen, lazy=True):
        from allura.model.repo import CommitDoc
        ci_doc = CommitDoc.m.get(_id=oid)
        if ci_doc and lazy: return False
        obj = self._hg[oid]
        # Save commit metadata
        mo = self.re_hg_user.match(obj.user())
        if mo:
            user_name, user_email = mo.groups()
        else:
            user_name = user_email = obj.user()
        user = Object(
            name=h.really_unicode(user_name),
            email=h.really_unicode(user_email),
            date=datetime.utcfromtimestamp(sum(obj.date())))
        fake_tree = self._tree_from_changectx(obj)
        args = dict(
            tree_id=fake_tree.hex(),
            committed=user,
            authored=user,
            message=h.really_unicode(obj.description() or ''),
            child_ids=[],
            parent_ids=[ p.hex() for p in self.real_parents(obj)
                                 if p.hex() != obj.hex() ])
        if ci_doc:
            ci_doc.update(args)
            ci_doc.m.save()
        else:
            ci_doc = CommitDoc(dict(args, _id=oid))
            try:
                ci_doc.m.insert(safe=True)
            except DuplicateKeyError:
                if lazy: return False
        self.refresh_tree_info(fake_tree, seen, lazy)
        return True

    def refresh_tree_info(self, tree, seen, lazy=True):
        from allura.model.repo import TreeDoc
        if lazy and tree.hex() in seen: return
        seen.add(tree.hex())
        doc = TreeDoc(dict(
                _id=tree.hex(),
                tree_ids=[],
                blob_ids=[],
                other_ids=[]))
        for name, t in tree.trees.iteritems():
            self.refresh_tree_info(t, seen, lazy)
            doc.tree_ids.append(
                dict(name=name, id=t.hex()))
        for name, oid in tree.blobs.iteritems():
            doc.blob_ids.append(
                dict(name=name, id=oid))
        doc.m.save(safe=False)
        return doc

    def log(self, object_id, skip, count):<commit_msg>[#4583] Handle strange encodings in mercurial repos<commit_after>import os
import re
import shutil
import logging
from binascii import b2a_hex
from datetime import datetime
from cStringIO import StringIO
from ConfigParser import ConfigParser

import tg
os.environ['HGRCPATH'] = '' # disable loading .hgrc
from mercurial import ui, hg
from pymongo.errors import DuplicateKeyError

from ming.base import Object
from ming.orm import Mapper, session
from ming.utils import LazyProperty

from allura import model as M
from allura.lib import helpers as h
from allura.model.repository import topological_sort, GitLikeTree

log = logging.getLogger(__name__)

class Repository(M.Repository):
    tool_name='Hg'
    repo_id='hg'
    type_s='Hg Repository'
    class __mongometa__:
        name='hg-repository'

    @LazyProperty
    def _impl(self):
        return HgImplementation(self)

    def merge_command(self, merge_request):
        '''Return the command to merge a given commit into a given target branch'''
        return 'hg checkout %s\nhg pull -r %s %s' % (
            merge_request.target_branch,
            merge_request.downstream.commit_id,
            merge_request.downstream_repo_url,
        )

    def count(self, branch='default'):
        return super(Repository, self).count(branch)

    def log(self, branch='default', offset=0, limit=10):
        return super(Repository, self).log(branch, offset, limit)

class HgImplementation(M.RepositoryImplementation):
    re_hg_user = re.compile('(.*) <(.*)>')

    def __init__(self, repo):
        self._repo = repo

    @LazyProperty
    def _hg(self):
        return hg.repository(ui.ui(), self._repo.full_fs_path)

    def init(self):
        fullname = self._setup_paths()
        log.info('hg init %s', fullname)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        repo = hg.repository(
            ui.ui(), self._repo.full_fs_path, create=True)
        self.__dict__['_hg'] = repo
        self._setup_special_files()
        self._repo.status = 'ready'

    def clone_from(self, source_url):
        '''Initialize a repo as a clone of another'''
        fullname = self._setup_paths(create_repo_dir=False)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        log.info('Initialize %r as a clone of %s',
                 self._repo, source_url)
        # !$ hg doesn't like unicode as urls
        src, repo = hg.clone(
            ui.ui(),
            source_url.encode('utf-8'),
            self._repo.full_fs_path.encode('utf-8'),
            update=False)
        self.__dict__['_hg'] = repo
        self._setup_special_files()
        self._repo.status = 'analyzing'
        session(self._repo).flush()
        log.info('... %r cloned, analyzing', self._repo)
        self._repo.refresh(notify=False)
        self._repo.status = 'ready'
        log.info('... %s ready', self._repo)
        session(self._repo).flush()

    def commit(self, rev):
        '''Return a Commit object.  rev can be _id or a branch/tag name'''
        # See if the rev is a named ref that we have cached, and use the sha1
        # from the cache. This ensures that we don't return a sha1 that we
        # don't have indexed into mongo yet.
        for ref in self._repo.heads + self._repo.branches + self._repo.repo_tags:
            if ref.name == rev:
                rev = ref.object_id
                break
        result = M.repo.Commit.query.get(_id=rev)
        if result is None:
            try:
                impl = self._hg[str(rev)]
                result = M.repo.Commit.query.get(_id=impl.hex())
            except Exception, e:
                log.exception(e)
        if result is None: return None
        result.set_context(self._repo)
        return result

    def real_parents(self, ci):
        """Return all parents of a commit, excluding the 'null revision' (a
        fake revision added as the parent of the root commit by the Hg api).
        """
        return [p for p in ci.parents() if p]

    def all_commit_ids(self):
        """Return a list of commit ids, starting with the root (first commit)
        of the tree and ending with the head(s).

        NB: The ForgeGit implementation returns commits in the opposite order.
        """
        graph = {}
        to_visit = [ self._hg[hd] for hd in self._hg.heads() ]
        while to_visit:
            obj = to_visit.pop()
            if obj.hex() in graph: continue
            parents = self.real_parents(obj)
            graph[obj.hex()] = set(
                p.hex() for p in parents
                if p.hex() != obj.hex())
            to_visit += parents
        return [ ci for ci in topological_sort(graph) ]

    def new_commits(self, all_commits=False):
        graph = {}
        to_visit = [ self._hg[hd] for hd in self._hg.heads() ]
        while to_visit:
            obj = to_visit.pop()
            if obj.hex() in graph: continue
            if not all_commits:
                # Look up the object
                if M.repo.Commit.query.find(dict(_id=obj.hex())).count():
                    graph[obj.hex()] = set() # mark as parentless
                    continue
            parents = self.real_parents(obj)
            graph[obj.hex()] = set(
                p.hex() for p in parents
                if p.hex() != obj.hex())
            to_visit += parents
        return list(topological_sort(graph))

    def refresh_heads(self):
        self._repo.heads = [
            Object(name=None, object_id=self._hg[head].hex())
            for head in self._hg.heads() ]
        self._repo.branches = [
            Object(name=name, object_id=self._hg[tag].hex())
            for name, tag in self._hg.branchtags().iteritems() ]
        self._repo.repo_tags = [
            Object(name=name, object_id=self._hg[tag].hex())
            for name, tag in self._hg.tags().iteritems() ]
        session(self._repo).flush()

    def refresh_commit_info(self, oid, seen, lazy=True):
        from allura.model.repo import CommitDoc
        ci_doc = CommitDoc.m.get(_id=oid)
        if ci_doc and lazy: return False
        obj = self._hg[oid]
        # Save commit metadata
        mo = self.re_hg_user.match(obj.user())
        if mo:
            user_name, user_email = mo.groups()
        else:
            user_name = user_email = obj.user()
        user = Object(
            name=h.really_unicode(user_name),
            email=h.really_unicode(user_email),
            date=datetime.utcfromtimestamp(sum(obj.date())))
        fake_tree = self._tree_from_changectx(obj)
        args = dict(
            tree_id=fake_tree.hex(),
            committed=user,
            authored=user,
            message=h.really_unicode(obj.description() or ''),
            child_ids=[],
            parent_ids=[ p.hex() for p in self.real_parents(obj)
                                 if p.hex() != obj.hex() ])
        if ci_doc:
            ci_doc.update(args)
            ci_doc.m.save()
        else:
            ci_doc = CommitDoc(dict(args, _id=oid))
            try:
                ci_doc.m.insert(safe=True)
            except DuplicateKeyError:
                if lazy: return False
        self.refresh_tree_info(fake_tree, seen, lazy)
        return True

    def refresh_tree_info(self, tree, seen, lazy=True):
        from allura.model.repo import TreeDoc
        if lazy and tree.hex() in seen: return
        seen.add(tree.hex())
        doc = TreeDoc(dict(
                _id=tree.hex(),
                tree_ids=[],
                blob_ids=[],
                other_ids=[]))
        for name, t in tree.trees.iteritems():
            self.refresh_tree_info(t, seen, lazy)
            doc.tree_ids.append(
                dict(name=h.really_unicode(name), id=t.hex()))
        for name, oid in tree.blobs.iteritems():
            doc.blob_ids.append(
                dict(name=h.really_unicode(name), id=oid))
        doc.m.save(safe=False)
        return doc

    def log(self, object_id, skip, count):
{'fraction_non_alphanumeric': 0.1093221400826964, 'fraction_numerical': 0.001252975817566721, 'mean_word_length': 1.9865294667913937, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1751968', 'n_tokens_mistral': 4840, 'n_tokens_neox': 4667, 'n_words': 1235}