<commit_before>#!/usr/bin/env python
# encoding: utf-8

from __future__ import division
from __future__ import unicode_literals

import re
import copy
import math
import logging
import progressbar

from pymongo.errors import DuplicateKeyError

from modularodm import Q
from modularodm.exceptions import NoResultsFound
from modularodm.storage.base import KeyExistsException

from framework.mongo import database
from framework.analytics import clean_page
from framework.transactions.context import TokuTransaction

from scripts import utils as scripts_utils

from website.app import init_app
from website.project.model import NodeLog
from website.addons.osfstorage import model, oldels

logger = logging.getLogger(__name__)


LOG_ACTIONS = set([
    'osf_storage_file_added',
    'osf_storage_file_updated',
    'osf_storage_file_removed',
    'osf_storage_file_restored',
    'file_added',
    'file_updated',
    'file_removed',
    'file_restored',
])


def migrate_download_counts(node, children, dry=True):
    collection = database['pagecounters']

    for old_path, new in children.items():
        if dry:
            new_id = ':'.join(['download', node._id, 'new_id'])
            old_id = ':'.join(['download', node._id, old_path])
        else:
            new_id = ':'.join(['download', node._id, new._id])
            old_id = ':'.join(['download', node._id, old_path])

        result = collection.find_one(
            {'_id': clean_page(old_id)},
            {'total': 1, 'unique': 1}
        )

        if not result:
            continue

        logger.info('Copying download counts of {!r} to {!r}'.format(old_path, new))

        if not dry:
            result['_id'] = new_id
            try:
                database.pagecounters.insert(result)
            except DuplicateKeyError:
                logger.warn('Already migrated {!r}'.format(old_path))
                continue
        else:
            continue

        for idx in range(len(new.versions)):
            result = collection.find_one(
                {'_id': clean_page('{}:{}'.format(old_id, idx + 1))},
                {'total': 1, 'unique': 1}
            )
            if not result:
                continue

            logger.info('Copying download count of version {} of {!r} to version {} of {!r}'.format(idx + 1, old_path, idx, new))
            if not dry:
                result['_id'] = '{}:{}'.format(new_id, idx)
                database.pagecounters.insert(result)


def migrate_node_settings(node_settings, dry=True):
    logger.info('Running `on add` for node settings of {}'.format(node_settings.owner._id))

    if not dry:
        node_settings.on_add()


def migrate_file(node, old, parent, dry=True):
    assert isinstance(old, oldels.OsfStorageFileRecord)
    if not dry:
        try:
            new = parent.append_file(old.name)
            logger.debug('Created new child {}'.format(old.name))
        except KeyExistsException:
            logger.warning('{!r} has already been migrated'.format(old))
            return parent.find_child_by_name(old.name)
        new.versions = old.versions
        new.is_deleted = old.is_deleted
        new.save()
    else:
        new = None
    return new


def migrate_logs(node, children, dry=True):
    res = list(NodeLog.find(
        Q('params.node', 'eq', node._id) |
        Q('params.project', 'eq', node._id)
    ))

    for log in res:
        if log.action not in LOG_ACTIONS:
            continue

        if dry:
            logger.debug('{!r} {} -> {}'.format(log, log.params['path'], 'New path'))
            continue

        try:
            new = children[log.params['path']]
        except KeyError:
            if not log.params['path'].startswith('/'):
                logger.warning('Failed to migrate log with path {}'.format(log.params['path']))
            continue

        mpath = new.materialized_path()
        url = '/{}/files/osfstorage/{}/'.format(node._id, new._id)
        logger.debug('{!r} {} -> {}'.format(log, log.params['path'], mpath))

        log.params['path'] = mpath
        log.params['urls'] = {
            'view': url,
            'download': url + '?action=download'
        }

        log.save()

    NodeLog._cache.clear()<commit_msg>Only match on node<commit_after>#!/usr/bin/env python
# encoding: utf-8

from __future__ import division
from __future__ import unicode_literals

import re
import copy
import math
import logging
import progressbar

from pymongo.errors import DuplicateKeyError

from modularodm import Q
from modularodm.exceptions import NoResultsFound
from modularodm.storage.base import KeyExistsException

from framework.mongo import database
from framework.analytics import clean_page
from framework.transactions.context import TokuTransaction

from scripts import utils as scripts_utils

from website.app import init_app
from website.project.model import NodeLog
from website.addons.osfstorage import model, oldels

logger = logging.getLogger(__name__)


LOG_ACTIONS = set([
    'osf_storage_file_added',
    'osf_storage_file_updated',
    'osf_storage_file_removed',
    'osf_storage_file_restored',
    'file_added',
    'file_updated',
    'file_removed',
    'file_restored',
])


def migrate_download_counts(node, children, dry=True):
    collection = database['pagecounters']

    for old_path, new in children.items():
        if dry:
            new_id = ':'.join(['download', node._id, 'new_id'])
            old_id = ':'.join(['download', node._id, old_path])
        else:
            new_id = ':'.join(['download', node._id, new._id])
            old_id = ':'.join(['download', node._id, old_path])

        result = collection.find_one(
            {'_id': clean_page(old_id)},
            {'total': 1, 'unique': 1}
        )

        if not result:
            continue

        logger.info('Copying download counts of {!r} to {!r}'.format(old_path, new))

        if not dry:
            result['_id'] = new_id
            try:
                database.pagecounters.insert(result)
            except DuplicateKeyError:
                logger.warn('Already migrated {!r}'.format(old_path))
                continue
        else:
            continue

        for idx in range(len(new.versions)):
            result = collection.find_one(
                {'_id': clean_page('{}:{}'.format(old_id, idx + 1))},
                {'total': 1, 'unique': 1}
            )
            if not result:
                continue

            logger.info('Copying download count of version {} of {!r} to version {} of {!r}'.format(idx + 1, old_path, idx, new))
            if not dry:
                result['_id'] = '{}:{}'.format(new_id, idx)
                database.pagecounters.insert(result)


def migrate_node_settings(node_settings, dry=True):
    logger.info('Running `on add` for node settings of {}'.format(node_settings.owner._id))

    if not dry:
        node_settings.on_add()


def migrate_file(node, old, parent, dry=True):
    assert isinstance(old, oldels.OsfStorageFileRecord)
    if not dry:
        try:
            new = parent.append_file(old.name)
            logger.debug('Created new child {}'.format(old.name))
        except KeyExistsException:
            logger.warning('{!r} has already been migrated'.format(old))
            return parent.find_child_by_name(old.name)
        new.versions = old.versions
        new.is_deleted = old.is_deleted
        new.save()
    else:
        new = None
    return new


def migrate_logs(node, children, dry=True):
    res = list(NodeLog.find(Q('params.node', 'eq', node._id)))

    for log in res:
        if log.action not in LOG_ACTIONS:
            continue

        if dry:
            logger.debug('{!r} {} -> {}'.format(log, log.params['path'], 'New path'))
            continue

        try:
            new = children[log.params['path']]
        except KeyError:
            if not log.params['path'].startswith('/'):
                logger.warning('Failed to migrate log with path {}'.format(log.params['path']))
            continue

        mpath = new.materialized_path()
        url = '/{}/files/osfstorage/{}/'.format(node._id, new._id)
        logger.debug('{!r} {} -> {}'.format(log, log.params['path'], mpath))

        log.params['path'] = mpath
        log.params['urls'] = {
            'view': url,
            'download': url + '?action=download'
        }

        log.save()

    NodeLog._cache.clear()
{'fraction_non_alphanumeric': 0.12611223158144502, 'fraction_numerical': 0.0016609324949578834, 'mean_word_length': 2.369304556354916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5312990', 'n_tokens_mistral': 2553, 'n_tokens_neox': 2462, 'n_words': 604}