<commit_before>import logging
from itertools import chain
from cPickle import dumps
import re

import bson

import tg

from ming.base import Object
from ming.orm import mapper, session

from allura.lib import utils
from allura.lib import helpers as h
from allura.model.repo import CommitDoc, TreeDoc, TreesDoc, DiffInfoDoc
from allura.model.repo import LastCommitDoc, CommitRunDoc
from allura.model.repo import Commit
from allura.model.index import ArtifactReferenceDoc, ShortlinkDoc

log = logging.getLogger(__name__)

QSIZE=100

def refresh_repo(repo, all_commits=False, notify=True):
    all_commit_ids = commit_ids = list(repo.all_commit_ids())
    new_commit_ids = unknown_commit_ids(commit_ids)
    stats_log = h.log_action(log, 'commit')
    for ci in new_commit_ids:
        stats_log.info(
            '',
            meta=dict(
                module='scm-%s' % repo.repo_id,
                read='0'))
    if not all_commits:
        # Skip commits that are already in the DB
        commit_ids = new_commit_ids
    log.info('Refreshing %d commits on %s', len(commit_ids), repo.full_fs_path)

    # Refresh commits
    seen = set()
    for i, oid in enumerate(commit_ids):
        repo.refresh_commit_info(oid, seen, not all_commits)
        if (i+1) % 100 == 0:
            log.info('Refresh commit info %d: %s', (i+1), oid)

    refresh_commit_repos(all_commit_ids, repo)

    # Refresh child references
    for i, oid in enumerate(commit_ids):
        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
        refresh_children(ci)
        if (i+1) % 100 == 0:
            log.info('Refresh child info %d for parents of %s', (i+1), ci._id)

    # Refresh commit runs
    rb = CommitRunBuilder(commit_ids)
    rb.run()
    rb.cleanup()

    # Refresh trees
    cache = {}
    for i, oid in enumerate(commit_ids):
        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
        cache = refresh_commit_trees(ci, cache)
        if (i+1) % 100 == 0:
            log.info('Refresh commit trees %d: %s', (i+1), ci._id)

    # Compute diffs
    cache = {}
    # Have to compute_diffs() for all commits to ensure that LastCommitDocs
    # are set properly for forked repos. For SVN, compute_diffs() will return
    # immediately, so no real work is done.
    for i, oid in enumerate(reversed(all_commit_ids)):
        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
        compute_diffs(repo._id, cache, ci)
        if (i+1) % 100 == 0:
            log.info('Compute diffs %d: %s', (i+1), ci._id)

    log.info('Refresh complete for %s', repo.full_fs_path)

    # Send notifications
    if notify:
        send_notifications(repo, commit_ids)

def refresh_commit_trees(ci, cache):
    '''Refresh the list of trees included withn a commit'''
    if ci.tree_id is None: return
    trees_doc = TreesDoc(dict(
            _id=ci._id,
            tree_ids = list(trees(ci.tree_id, cache))))
    trees_doc.m.save(safe=False)
    new_cache = dict(
        (oid, cache[oid])
        for oid in trees_doc.tree_ids)<commit_msg>[#4560] Fixed error when refreshing SVN repos<commit_after>import logging
from itertools import chain
from cPickle import dumps
import re

import bson

import tg

from ming.base import Object
from ming.orm import mapper, session

from allura.lib import utils
from allura.lib import helpers as h
from allura.model.repo import CommitDoc, TreeDoc, TreesDoc, DiffInfoDoc
from allura.model.repo import LastCommitDoc, CommitRunDoc
from allura.model.repo import Commit
from allura.model.index import ArtifactReferenceDoc, ShortlinkDoc

log = logging.getLogger(__name__)

QSIZE=100

def refresh_repo(repo, all_commits=False, notify=True):
    all_commit_ids = commit_ids = list(repo.all_commit_ids())
    new_commit_ids = unknown_commit_ids(commit_ids)
    stats_log = h.log_action(log, 'commit')
    for ci in new_commit_ids:
        stats_log.info(
            '',
            meta=dict(
                module='scm-%s' % repo.repo_id,
                read='0'))
    if not all_commits:
        # Skip commits that are already in the DB
        commit_ids = new_commit_ids
    log.info('Refreshing %d commits on %s', len(commit_ids), repo.full_fs_path)

    # Refresh commits
    seen = set()
    for i, oid in enumerate(commit_ids):
        repo.refresh_commit_info(oid, seen, not all_commits)
        if (i+1) % 100 == 0:
            log.info('Refresh commit info %d: %s', (i+1), oid)

    refresh_commit_repos(all_commit_ids, repo)

    # Refresh child references
    for i, oid in enumerate(commit_ids):
        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
        refresh_children(ci)
        if (i+1) % 100 == 0:
            log.info('Refresh child info %d for parents of %s', (i+1), ci._id)

    # Refresh commit runs
    rb = CommitRunBuilder(commit_ids)
    rb.run()
    rb.cleanup()

    # Refresh trees
    cache = {}
    for i, oid in enumerate(commit_ids):
        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
        cache = refresh_commit_trees(ci, cache)
        if (i+1) % 100 == 0:
            log.info('Refresh commit trees %d: %s', (i+1), ci._id)

    # Compute diffs
    cache = {}
    # Have to compute_diffs() for all commits to ensure that LastCommitDocs
    # are set properly for forked repos. For SVN, compute_diffs() we don't
    # want to pre-compute the diffs because that would be too expensive, so
    # we skip them here and do them on-demand with caching.
    if repo.tool.lower() != 'svn':
        for i, oid in enumerate(reversed(all_commit_ids)):
            ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
            compute_diffs(repo._id, cache, ci)
            if (i+1) % 100 == 0:
                log.info('Compute diffs %d: %s', (i+1), ci._id)

    log.info('Refresh complete for %s', repo.full_fs_path)

    # Send notifications
    if notify:
        send_notifications(repo, commit_ids)

def refresh_commit_trees(ci, cache):
    '''Refresh the list of trees included withn a commit'''
    if ci.tree_id is None: return
    trees_doc = TreesDoc(dict(
            _id=ci._id,
            tree_ids = list(trees(ci.tree_id, cache))))
    trees_doc.m.save(safe=False)
    new_cache = dict(
        (oid, cache[oid])
        for oid in trees_doc.tree_ids)
{'fraction_non_alphanumeric': 0.10814249363867684, 'fraction_numerical': 0.009541984732824428, 'mean_word_length': 2.889301175015461, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4209715', 'n_tokens_mistral': 2131, 'n_tokens_neox': 2037, 'n_words': 581}