<commit_before>import os
import shutil
import string
import logging
import subprocess
from hashlib import sha1
from cStringIO import StringIO
from datetime import datetime

import tg
import pysvn
from pylons import c

from ming.base import Object
from ming.orm import MappedClass, FieldProperty, session
from ming.utils import LazyProperty

from allura import model as M
from allura.lib import helpers as h
from allura.model.repository import GitLikeTree

log = logging.getLogger(__name__)

class Repository(M.Repository):
    tool_name='SVN'
    repo_id='svn'
    type_s='SVN Repository'
    class __mongometa__:
        name='svn-repository'
    branches = FieldProperty([dict(name=str,object_id=str)])

    def __init__(self, **kw):
        super(Repository, self).__init__(**kw)
        self._impl = SVNImplementation(self)

    def readonly_clone_command(self):
        return 'svn checkout svn://%s' % self.scm_url_path

    def readwrite_clone_command(self):
        return 'svn checkout svn+ssh://%s@%s' % (c.user.username, self.scm_url_path)

    def _log(self, rev, skip, max_count):
        ci = self.commit(rev)
        if ci is None: return []
        return ci.log(int(skip), int(max_count))

    def compute_diffs(self): return

    def count(self, *args, **kwargs):
        return super(Repository, self).count(None)

    def log(self, branch=None, offset=0, limit=10):
        return list(self._log(rev=branch, skip=offset, max_count=limit))

    def latest(self, branch=None):
        if self._impl is None: return None
        if not self.heads: return None
        return self._impl.commit(self.heads[0].object_id)

class SVNImplementation(M.RepositoryImplementation):
    post_receive_template = string.Template(
        '#!/bin/bash\n'
        'curl -s $url\n')

    def __init__(self, repo):
        self._repo = repo

    @LazyProperty
    def _svn(self):
        return pysvn.Client()

    @LazyProperty
    def _url(self):
        return 'file://%s/%s' % (self._repo.fs_path, self._repo.name)

    def shorthand_for_commit(self, commit):
        return '[r%d]' % self._revno(commit.object_id)

    def url_for_commit(self, commit):
        return '%s%d/' % (
            self._repo.url(), self._revno(commit.object_id))

    def init(self):
        fullname = self._setup_paths()
        log.info('svn init %s', fullname)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        subprocess.call(['svnadmin', 'create', self._repo.name],
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 cwd=self._repo.fs_path)
        self._setup_special_files()
        self._repo.status = 'ready'

    def refresh_heads(self):
        info = self._svn.info2(
            self._url,
            revision=pysvn.Revision(pysvn.opt_revision_kind.head),
            recurse=False)[0][1]
        oid = self._oid(info.rev.number)
        self._repo.heads = [ Object(name=None, object_id=oid) ]
        # Branches and tags aren't really supported in subversion
        self._repo.branches = []
        self._repo.repo_tags = []
        session(self._repo).flush()

    def commit(self, rev):
        if rev in ('HEAD', None):
            if not self._repo.heads: return None
            oid = self._repo.heads[0].object_id
        elif isinstance(rev, int) or rev.isdigit():
            oid = self._oid(rev)
        else:
            oid = rev
        result = M.Commit.query.get(object_id=oid)
        if result is None: return None
        result.set_context(self._repo)
        return result

    def new_commits(self, all_commits=False):
        head_revno = self._revno(self._repo.heads[0].object_id)
        oids = [ self._oid(revno) for revno in range(1, head_revno+1) ]
        if all_commits:
            return oids
        # Find max commit id -- everything greater than that will be "unknown"
        q = M.Commit.query.find(dict(type='commit')).sort('-object_id')
        last_commit = q.first()
        if last_commit is None: return oids
        return [
            oid for oid in oids if oid > last_commit.object_id ]

    def commit_context(self, commit):
        revno = int(commit.object_id.split(':')[1])
        prev,next=[],[]
        if revno > 1:
            prev = [ self.commit(revno - 1) ]
        if revno < self._revno(self._repo.heads[0].object_id):<commit_msg>[#1291] Fix sorting of svn commit ids<commit_after>import os
import shutil
import string
import logging
import subprocess
from hashlib import sha1
from cStringIO import StringIO
from datetime import datetime

import tg
import pymongo
import pysvn
from pylons import c

from ming.base import Object
from ming.orm import MappedClass, FieldProperty, session
from ming.utils import LazyProperty

from allura import model as M
from allura.lib import helpers as h
from allura.model.repository import GitLikeTree

log = logging.getLogger(__name__)

class Repository(M.Repository):
    tool_name='SVN'
    repo_id='svn'
    type_s='SVN Repository'
    class __mongometa__:
        name='svn-repository'
    branches = FieldProperty([dict(name=str,object_id=str)])

    def __init__(self, **kw):
        super(Repository, self).__init__(**kw)
        self._impl = SVNImplementation(self)

    def readonly_clone_command(self):
        return 'svn checkout svn://%s' % self.scm_url_path

    def readwrite_clone_command(self):
        return 'svn checkout svn+ssh://%s@%s' % (c.user.username, self.scm_url_path)

    def _log(self, rev, skip, max_count):
        ci = self.commit(rev)
        if ci is None: return []
        return ci.log(int(skip), int(max_count))

    def compute_diffs(self): return

    def count(self, *args, **kwargs):
        return super(Repository, self).count(None)

    def log(self, branch=None, offset=0, limit=10):
        return list(self._log(rev=branch, skip=offset, max_count=limit))

    def latest(self, branch=None):
        if self._impl is None: return None
        if not self.heads: return None
        return self._impl.commit(self.heads[0].object_id)

class SVNImplementation(M.RepositoryImplementation):
    post_receive_template = string.Template(
        '#!/bin/bash\n'
        'curl -s $url\n')

    def __init__(self, repo):
        self._repo = repo

    @LazyProperty
    def _svn(self):
        return pysvn.Client()

    @LazyProperty
    def _url(self):
        return 'file://%s/%s' % (self._repo.fs_path, self._repo.name)

    def shorthand_for_commit(self, commit):
        return '[r%d]' % self._revno(commit.object_id)

    def url_for_commit(self, commit):
        return '%s%d/' % (
            self._repo.url(), self._revno(commit.object_id))

    def init(self):
        fullname = self._setup_paths()
        log.info('svn init %s', fullname)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        subprocess.call(['svnadmin', 'create', self._repo.name],
                                 stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE,
                                 cwd=self._repo.fs_path)
        self._setup_special_files()
        self._repo.status = 'ready'

    def refresh_heads(self):
        info = self._svn.info2(
            self._url,
            revision=pysvn.Revision(pysvn.opt_revision_kind.head),
            recurse=False)[0][1]
        oid = self._oid(info.rev.number)
        self._repo.heads = [ Object(name=None, object_id=oid) ]
        # Branches and tags aren't really supported in subversion
        self._repo.branches = []
        self._repo.repo_tags = []
        session(self._repo).flush()

    def commit(self, rev):
        if rev in ('HEAD', None):
            if not self._repo.heads: return None
            oid = self._repo.heads[0].object_id
        elif isinstance(rev, int) or rev.isdigit():
            oid = self._oid(rev)
        else:
            oid = rev
        result = M.Commit.query.get(object_id=oid)
        if result is None: return None
        result.set_context(self._repo)
        return result

    def new_commits(self, all_commits=False):
        head_revno = self._revno(self._repo.heads[0].object_id)
        oids = [ self._oid(revno) for revno in range(1, head_revno+1) ]
        if all_commits:
            return oids
        # Find max commit id -- everything greater than that will be "unknown"
        q = M.Commit.query.find(
            dict(type='commit')).sort('object_id', pymongo.DESCENDING)
        last_commit = q.first()
        if last_commit is None: return oids
        return [
            oid for oid in oids if oid > last_commit.object_id ]

    def commit_context(self, commit):
        revno = int(commit.object_id.split(':')[1])
        prev,next=[],[]
        if revno > 1:
            prev = [ self.commit(revno - 1) ]
        if revno < self._revno(self._repo.heads[0].object_id):
{'fraction_non_alphanumeric': 0.11292290626733223, 'fraction_numerical': 0.003993344425956739, 'mean_word_length': 2.413858386974631, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '751555', 'n_tokens_mistral': 2891, 'n_tokens_neox': 2804, 'n_words': 657}