<commit_before>import os
import stat
import errno
import mimetypes
import logging
from hashlib import sha1
from datetime import datetime
from collections import defaultdict

import pylons
from tg import config
import pymongo.errors

from ming import schema as S
from ming.utils import LazyProperty
from ming.orm import MappedClass, FieldProperty, session, mapper

from allura.lib.patience import SequenceMatcher
from allura.lib import helpers as h

from .artifact import Artifact, VersionedArtifact
from .auth import User
from .session import repository_orm_session, project_orm_session

log = logging.getLogger(__name__)

class RepositoryImplementation(object):

    # Repository-specific code
    def init(self): # pragma no cover
        raise NotImplementedError, 'init'

    def clone_from(self, source_path): # pragma no cover
        raise NotImplementedError, 'clone_from'

    def commit(self, revision): # pragma no cover
        raise NotImplementedError, 'commit'

    def new_commits(self, all_commits=False): # pragma no cover
        '''Return any commit object_ids in the native repo that are not (yet) stored
        in the database in topological order (parents first)'''
        raise NotImplementedError, 'commit'

    def commit_context(self, object_id): # pragma no cover
        '''Returns {'prev':Commit, 'next':Commit}'''
        raise NotImplementedError, 'context'

    def refresh_heads(self): # pragma no cover
        '''Sets repository metadata such as heads, tags, and branches'''
        raise NotImplementedError, 'refresh_heads'

    def refresh_commit(self, ci, seen_object_ids): # pragma no cover
        '''Refresh the data in the commit object 'ci' with data from the repo'''
        raise NotImplementedError, 'refresh_commit'

    def _setup_receive_hook(self): # pragma no cover
        '''Install a hook in the repository that will ping the refresh url for
        the repo'''
        raise NotImplementedError, '_setup_receive_hook'

    def log(self, object_id, skip, count): # pragma no cover
        '''Return a list of object_ids beginning at the given commit ID and continuing
        to the parent nodes in a breadth-first traversal.  Also return a list of 'next commit' options
        (these are candidates for he next commit after 'count' commits have been
        exhausted).'''
        raise NotImplementedError, '_log'

    def compute_tree(self, commit, path='/'):
        '''Used in hg and svn to compute a git-like-tree lazily'''
        raise NotImplementedError, 'compute_tree'

    def open_blob(self, blob): # pragma no cover
        '''Return a file-like object that contains the contents of the blob'''
        raise NotImplementedError, 'open_blob'

    def shorthand_for_commit(self, commit):
        return '[%s]' % commit.object_id[:6]

    def url_for_commit(self, commit):
        return '%sci/%s/' % (self._repo.url(), commit.object_id)

    def _setup_paths(self, create_repo_dir=True):
        if not self._repo.fs_path.endswith('/'): self._repo.fs_path += '/'
        fullname = os.path.join(self._repo.fs_path, self._repo.name)
        try:
            os.makedirs(fullname if create_repo_dir else self._repo.fs_path)
        except OSError, e: # pragma no cover
            if e.errno != errno.EEXIST: raise
        return fullname

    def _setup_special_files(self):
        magic_file = os.path.join(self._repo.fs_path, self._repo.name, '.SOURCEFORGE-REPOSITORY')
        with open(magic_file, 'w') as f:
            f.write(self._repo.repo_id)
        os.chmod(magic_file, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
        self._setup_receive_hook()

class Repository(Artifact):
    BATCH_SIZE=100
    class __mongometa__:
        name='generic-repository'
    _impl = None
    repo_id='repo'
    type_s='Repository'

    name=FieldProperty(str)
    tool=FieldProperty(str)
    fs_path=FieldProperty(str)
    url_path=FieldProperty(str)
    status=FieldProperty(str)
    email_address=''
    additional_viewable_extensions=FieldProperty(str)
    heads = FieldProperty([dict(name=str,object_id=str, count=int)])
    branches = FieldProperty([dict(name=str,object_id=str, count=int)])
    repo_tags = FieldProperty([dict(name=str,object_id=str, count=int)])
    upstream_repo = FieldProperty(dict(name=str,url=str))

    def __init__(self, **kw):
        if 'name' in kw and 'tool' in kw:
            if 'fs_path' not in kw:
                kw['fs_path'] = '/' + os.path.join(
                    kw['tool'],
                    pylons.c.project.url()[1:])
            if 'url_path' not in kw:
                kw['url_path'] = pylons.c.project.url()
        super(Repository, self).__init__(**kw)

    def __repr__(self): # pragma no cover
        return '<%s %s>' % (
            self.__class__.__name__,
            self.full_fs_path)

    # Proxy to _impl
    def init(self):
        return self._impl.init()
    def commit(self, rev):
        return self._impl.commit(rev)
    def commit_context(self, commit):
        return self._impl.commit_context(commit)
    def open_blob(self, blob):
        return self._impl.open_blob(blob)
    def shorthand_for_commit(self, commit):
        return self._impl.shorthand_for_commit(commit)
    def url_for_commit(self, commit):
        return self._impl.url_for_commit(commit)
    def compute_tree(self, commit, path='/'):
        return self._impl.compute_tree(commit, path)

    def _log(self, rev, skip, max_count):
        ci = self.commit(rev)
        if ci is None: return []
        return ci.log(int(skip), int(max_count))

    def init_as_clone(self, source_path, source_name, source_url):
        self.upstream_repo.name = source_name
        self.upstream_repo.url = source_url
        session(self).flush(self)
        self._impl.clone_from(source_path)

    def log(self, branch='master', offset=0, limit=10):
        return list(self._log(rev=branch, skip=offset, max_count=limit))

    def count(self, branch='master'):
        try:
            ci = self.commit(branch)
            if ci is None: return 0
            return ci.count_revisions()
        except: # pragma no cover
            log.exception('Error getting repo count')
            return 0

    def latest(self, branch='master'):
        if self._impl is None: return None
        try:
            return self.commit(branch)
        except: # pragma no cover
            return None

    def url(self):
        return self.app_config.url()

    def shorthand_id(self):
        return self.name

    def index(self):
        result = Artifact.index(self)
        result.update(
            name_s=self.name,
            type_s=self.type_s)
        return result

    @property
    def full_fs_path(self):
        return os.path.join(self.fs_path, self.name)

    @property
    def scm_url_path(self):
        return self.scm_host() + self.url_path + self.name

    def scm_host(self):
        return self.tool + config.get('scm.host', '.' + pylons.request.host)

    def merge_requests_by_statuses(self, *statuses):
        return MergeRequest.query.find(dict(
                app_config_id=self.app.config._id,
                status={'$in':statuses})).sort(
            'request_number')

    @LazyProperty
    def _additional_viewable_extensions(self):
        ext_list = self.additional_viewable_extensions or ''
        ext_list = [ext.strip() for ext in ext_list.split(',') if ext]
        ext_list += [ '.ini', '.gitignore', '.svnignore', 'README' ]
        return ext_list

    def guess_type(self, name):
        '''Guess the mime type and encoding of a given filename'''
        content_type, encoding = mimetypes.guess_type(name)
        if content_type is None or not content_type.startswith('text/'):
            fn, ext = os.path.splitext(name)
            ext = ext or fn
            if ext in self._additional_viewable_extensions:
                content_type, encoding = 'text/plain', None
            if content_type is None:
                content_type, encoding = 'application/octet-stream', None
        return content_type, encoding

    def refresh(self, all_commits=False):
        '''Find any new commits in the repository and update'''
        self._impl.refresh_heads()
        self.status = 'analyzing'
        session(self).flush()
        sess = session(Commit)
        log.info('Refreshing repository %s', self)
        commit_ids = self._impl.new_commits(all_commits)
        log.info('... %d new commits', len(commit_ids))
        # Refresh history
        i=0
        seen_object_ids = set()
        for i, oid in enumerate(commit_ids):
            if len(seen_object_ids) > 10000: # pragma no cover
                log.info('... flushing seen object cache')
                seen_object_ids = set()
            ci, isnew = Commit.upsert(oid)
            if not isnew and not all_commits:
                 # race condition, let the other proc handle it
                sess.expunge(ci)
                continue
            ci.set_context(self)
            self._impl.refresh_commit(ci, seen_object_ids)
            if (i+1) % self.BATCH_SIZE == 0:
                log.info('...... flushing %d commits (%d total)',
                         self.BATCH_SIZE, (i+1))
                sess.flush()
                sess.clear()
        log.info('...... flushing %d commits (%d total)',<commit_msg>[#1171] Added notification for commits to scm repos<commit_after>import os
import stat
import errno
import mimetypes
import logging
from hashlib import sha1
from datetime import datetime
from collections import defaultdict

import pylons
from tg import config
import pymongo.errors

from ming import schema as S
from ming.utils import LazyProperty
from ming.orm import MappedClass, FieldProperty, session, mapper

from allura.lib.patience import SequenceMatcher
from allura.lib import helpers as h

from .artifact import Artifact, VersionedArtifact, Feed
from .auth import User
from .session import repository_orm_session, project_orm_session
from .notification import Notification

log = logging.getLogger(__name__)
common_suffix = config.get('forgemail.domain', '.sourceforge.net')
common_prefix = config.get('forgemail.url', 'https://sourceforge.net')

class RepositoryImplementation(object):

    # Repository-specific code
    def init(self): # pragma no cover
        raise NotImplementedError, 'init'

    def clone_from(self, source_path): # pragma no cover
        raise NotImplementedError, 'clone_from'

    def commit(self, revision): # pragma no cover
        raise NotImplementedError, 'commit'

    def new_commits(self, all_commits=False): # pragma no cover
        '''Return any commit object_ids in the native repo that are not (yet) stored
        in the database in topological order (parents first)'''
        raise NotImplementedError, 'commit'

    def commit_context(self, object_id): # pragma no cover
        '''Returns {'prev':Commit, 'next':Commit}'''
        raise NotImplementedError, 'context'

    def refresh_heads(self): # pragma no cover
        '''Sets repository metadata such as heads, tags, and branches'''
        raise NotImplementedError, 'refresh_heads'

    def refresh_commit(self, ci, seen_object_ids): # pragma no cover
        '''Refresh the data in the commit object 'ci' with data from the repo'''
        raise NotImplementedError, 'refresh_commit'

    def _setup_receive_hook(self): # pragma no cover
        '''Install a hook in the repository that will ping the refresh url for
        the repo'''
        raise NotImplementedError, '_setup_receive_hook'

    def log(self, object_id, skip, count): # pragma no cover
        '''Return a list of object_ids beginning at the given commit ID and continuing
        to the parent nodes in a breadth-first traversal.  Also return a list of 'next commit' options
        (these are candidates for he next commit after 'count' commits have been
        exhausted).'''
        raise NotImplementedError, '_log'

    def compute_tree(self, commit, path='/'):
        '''Used in hg and svn to compute a git-like-tree lazily'''
        raise NotImplementedError, 'compute_tree'

    def open_blob(self, blob): # pragma no cover
        '''Return a file-like object that contains the contents of the blob'''
        raise NotImplementedError, 'open_blob'

    def shorthand_for_commit(self, commit):
        return '[%s]' % commit.object_id[:6]

    def url_for_commit(self, commit):
        return '%sci/%s/' % (self._repo.url(), commit.object_id)

    def _setup_paths(self, create_repo_dir=True):
        if not self._repo.fs_path.endswith('/'): self._repo.fs_path += '/'
        fullname = os.path.join(self._repo.fs_path, self._repo.name)
        try:
            os.makedirs(fullname if create_repo_dir else self._repo.fs_path)
        except OSError, e: # pragma no cover
            if e.errno != errno.EEXIST: raise
        return fullname

    def _setup_special_files(self):
        magic_file = os.path.join(self._repo.fs_path, self._repo.name, '.SOURCEFORGE-REPOSITORY')
        with open(magic_file, 'w') as f:
            f.write(self._repo.repo_id)
        os.chmod(magic_file, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
        self._setup_receive_hook()

class Repository(Artifact):
    BATCH_SIZE=100
    class __mongometa__:
        name='generic-repository'
    _impl = None
    repo_id='repo'
    type_s='Repository'

    name=FieldProperty(str)
    tool=FieldProperty(str)
    fs_path=FieldProperty(str)
    url_path=FieldProperty(str)
    status=FieldProperty(str)
    email_address=''
    additional_viewable_extensions=FieldProperty(str)
    heads = FieldProperty([dict(name=str,object_id=str, count=int)])
    branches = FieldProperty([dict(name=str,object_id=str, count=int)])
    repo_tags = FieldProperty([dict(name=str,object_id=str, count=int)])
    upstream_repo = FieldProperty(dict(name=str,url=str))

    def __init__(self, **kw):
        if 'name' in kw and 'tool' in kw:
            if 'fs_path' not in kw:
                kw['fs_path'] = '/' + os.path.join(
                    kw['tool'],
                    pylons.c.project.url()[1:])
            if 'url_path' not in kw:
                kw['url_path'] = pylons.c.project.url()
        super(Repository, self).__init__(**kw)

    def __repr__(self): # pragma no cover
        return '<%s %s>' % (
            self.__class__.__name__,
            self.full_fs_path)

    # Proxy to _impl
    def init(self):
        return self._impl.init()
    def commit(self, rev):
        return self._impl.commit(rev)
    def commit_context(self, commit):
        return self._impl.commit_context(commit)
    def open_blob(self, blob):
        return self._impl.open_blob(blob)
    def shorthand_for_commit(self, commit):
        return self._impl.shorthand_for_commit(commit)
    def url_for_commit(self, commit):
        return self._impl.url_for_commit(commit)
    def compute_tree(self, commit, path='/'):
        return self._impl.compute_tree(commit, path)

    def _log(self, rev, skip, max_count):
        ci = self.commit(rev)
        if ci is None: return []
        return ci.log(int(skip), int(max_count))

    def init_as_clone(self, source_path, source_name, source_url):
        self.upstream_repo.name = source_name
        self.upstream_repo.url = source_url
        session(self).flush(self)
        self._impl.clone_from(source_path)

    def log(self, branch='master', offset=0, limit=10):
        return list(self._log(rev=branch, skip=offset, max_count=limit))

    def count(self, branch='master'):
        try:
            ci = self.commit(branch)
            if ci is None: return 0
            return ci.count_revisions()
        except: # pragma no cover
            log.exception('Error getting repo count')
            return 0

    def latest(self, branch='master'):
        if self._impl is None: return None
        try:
            return self.commit(branch)
        except: # pragma no cover
            return None

    def url(self):
        return self.app_config.url()

    def shorthand_id(self):
        return self.name

    @property
    def email_address(self):
        domain = '.'.join(reversed(self.app.url[1:-1].split('/'))).replace('_', '-')
        return 'noreply@%s%s' % (domain, common_suffix)

    def index(self):
        result = Artifact.index(self)
        result.update(
            name_s=self.name,
            type_s=self.type_s)
        return result

    @property
    def full_fs_path(self):
        return os.path.join(self.fs_path, self.name)

    @property
    def scm_url_path(self):
        return self.scm_host() + self.url_path + self.name

    def scm_host(self):
        return self.tool + config.get('scm.host', '.' + pylons.request.host)

    def merge_requests_by_statuses(self, *statuses):
        return MergeRequest.query.find(dict(
                app_config_id=self.app.config._id,
                status={'$in':statuses})).sort(
            'request_number')

    @LazyProperty
    def _additional_viewable_extensions(self):
        ext_list = self.additional_viewable_extensions or ''
        ext_list = [ext.strip() for ext in ext_list.split(',') if ext]
        ext_list += [ '.ini', '.gitignore', '.svnignore', 'README' ]
        return ext_list

    def guess_type(self, name):
        '''Guess the mime type and encoding of a given filename'''
        content_type, encoding = mimetypes.guess_type(name)
        if content_type is None or not content_type.startswith('text/'):
            fn, ext = os.path.splitext(name)
            ext = ext or fn
            if ext in self._additional_viewable_extensions:
                content_type, encoding = 'text/plain', None
            if content_type is None:
                content_type, encoding = 'application/octet-stream', None
        return content_type, encoding

    def refresh(self, all_commits=False):
        '''Find any new commits in the repository and update'''
        self._impl.refresh_heads()
        self.status = 'analyzing'
        session(self).flush()
        sess = session(Commit)
        log.info('Refreshing repository %s', self)
        commit_ids = self._impl.new_commits(all_commits)
        log.info('... %d new commits', len(commit_ids))
        # Refresh history
        i=0
        seen_object_ids = set()
        commit_msgs = []
        for i, oid in enumerate(commit_ids):
            if len(seen_object_ids) > 10000: # pragma no cover
                log.info('... flushing seen object cache')
                seen_object_ids = set()
            ci, isnew = Commit.upsert(oid)
            if not isnew and not all_commits:
                 # race condition, let the other proc handle it
                sess.expunge(ci)
                continue
            ci.set_context(self)
            self._impl.refresh_commit(ci, seen_object_ids)
            if (i+1) % self.BATCH_SIZE == 0:
                log.info('...... flushing %d commits (%d total)',
                         self.BATCH_SIZE, (i+1))
                sess.flush()
                sess.clear()
            commit_msgs.append('%s <%s%s>' % (ci.summary,common_prefix,ci.url()))
        if not all_commits:
            Notification.post(
                artifact=self,
                topic='metadata',
                subject='New commit to %s %s' % (self.app.project.name,self.app.config.options.mount_label),
                text='\n'.join(commit_msgs))
        log.info('...... flushing %d commits (%d total)',
{'fraction_non_alphanumeric': 0.10543640384714294, 'fraction_numerical': 0.002365890037545646, 'mean_word_length': 2.588115888540321, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2248597', 'n_tokens_mistral': 5928, 'n_tokens_neox': 5713, 'n_words': 1533}