<commit_before>import json
import celery
from faker import Faker
import datetime
from modularodm import Q
import requests

import mock  # noqa
from mock import call
from nose.tools import *  # noqa PEP8 asserts
import httpretty

from scripts import cleanup_failed_registrations as scripts

from framework.auth import Auth
from framework.tasks import handlers

from website.archiver import (
    ARCHIVER_CHECKING,
    ARCHIVER_PENDING,
    ARCHIVER_SUCCESS,
    ARCHIVER_FAILURE,
    ARCHIVE_COPY_FAIL,
    ARCHIVE_SIZE_EXCEEDED,
)
from website.archiver import utils as archiver_utils
from website.app import *  # noqa
from website import archiver
from website.archiver import listeners
from website.archiver.tasks import *   # noqa

from website import mails
from website import settings
from website.util import waterbutler_url_for
from website.project.model import Node
from website.addons.base import StorageAddonBase

from tests import factories
from tests.base import OsfTestCase

fake = Faker()

FILE_TREE = {
    'path': '/',
    'name': '',
    'kind': 'folder',
    'children': [
        {
            'path': '/1234567',
            'name': 'Afile.file',
            'kind': 'file',
            'size': '128',
        },
        {
            'path': '/qwerty',
            'name': 'A Folder',
            'kind': 'folder',
            'children': [
                {
                    'path': '/qwerty/asdfgh',
                    'name': 'coolphoto.png',
                    'kind': 'file',
                    'size': '256',
                }
            ],
        }
    ],
}

class ArchiverTestCase(OsfTestCase):
    def setUp(self):
        super(ArchiverTestCase, self).setUp()
        handlers.celery_before_request()
        self.user = factories.UserFactory()
        self.auth = Auth(user=self.user)
        self.src = factories.NodeFactory(creator=self.user)
        self.src.add_addon('dropbox', auth=self.auth)
        self.dst = factories.RegistrationFactory(user=self.user, project=self.src, send_signals=False)
        self.stat_result = archiver_utils.aggregate_file_tree_metadata('dropbox', FILE_TREE, self.user)
        self.pks = (self.src._id, self.dst._id, self.user._id)

class TestStorageAddonBase(ArchiverTestCase):

    RESP_MAP = {
        '/': dict(data=FILE_TREE['children']),
        '/1234567': dict(data=FILE_TREE['children'][0]),
        '/qwerty': dict(data=FILE_TREE['children'][1]['children']),
        '/qwerty/asdfgh': dict(data=FILE_TREE['children'][1]['children'][0]),
    }

    @httpretty.activate
    def _test__get_file_tree(self, addon_short_name):
        requests_made = []
        def callback(request, uri, headers):
            path = request.querystring['path'][0]
            requests_made.append(path)
            return (200, headers, json.dumps(self.RESP_MAP[path]))

        for path in self.RESP_MAP.keys():
            url = waterbutler_url_for(
                'metadata',
                provider=addon_short_name,
                path=path,
                node=self.src,
                user=self.user,
                view_only=True,
            )
            httpretty.register_uri(httpretty.GET,
                                   url,
                                   body=callback,
                                   content_type='applcation/json')
        addon = self.src.get_or_add_addon(addon_short_name, auth=self.auth)
        root = {
            'path': '/',
            'name': '',
            'kind': 'folder',
        }
        file_tree = addon._get_file_tree(root, self.user)
        assert_equal(FILE_TREE, file_tree)
        assert_equal(requests_made, ['/', '/qwerty'])  # no requests made for files

    def _test_addon(self, addon_short_name):
        self._test__get_file_tree(addon_short_name)

    def test_addons(self):
        #  Test that each addon in settings.ADDONS_ARCHIVABLE if not addon == 'wiki' implements the StorageAddonBase interface
        for addon in [a for a in settings.ADDONS_ARCHIVABLE if not a == 'wiki']:
            self._test_addon(addon)

class TestArchiverTasks(ArchiverTestCase):

    @mock.patch('website.archiver.tasks.stat_node.delay')
    def test_archive(self, mock_stat):
        src_pk, dst_pk, user_pk = self.pks
        archive(src_pk, dst_pk, user_pk)
        mock_stat.assert_called_with(src_pk, dst_pk, user_pk)
        assert_true(self.dst.archiving)

    def test_stat_node(self):
        src_pk, dst_pk, user_pk = self.pks
        self.src.add_addon('box', auth=self.auth)  # has box, dropbox
        with mock.patch('celery.group') as mock_group:
            stat_node(src_pk, dst_pk, user_pk)
        stat_dropbox_sig = stat_addon.si('dropbox', src_pk, dst_pk, user_pk)
        stat_box_sig = stat_addon.si('box', src_pk, dst_pk, user_pk)
        assert(mock_group.called_with(stat_dropbox_sig, stat_box_sig))

    def test_stat_addon(self):
        src_pk, dst_pk, user_pk = self.pks
        with mock.patch.object(StorageAddonBase, '_get_file_tree') as mock_file_tree:
            with mock.patch.object(archiver, 'AggregateStatResult') as MockStat:
                mock_file_tree.return_value = FILE_TREE
                res = stat_addon('dropbox', src_pk,  dst_pk, user_pk)
        assert_equal(self.dst.archived_providers['dropbox']['status'], ARCHIVER_CHECKING)
        src_dropbox = self.src.get_addon('dropbox')
        assert(MockStat.called_with(
            src_dropbox._id,
            'dropbox',
            targets=[archiver_utils.aggregate_file_tree_metadata(src_dropbox, FILE_TREE, self.user)]
        ))
        assert_equal(res.target_name, 'dropbox')

    def test_archive_node_pass(self):
        src_pk, dst_pk, user_pk = self.pks
        with mock.patch.object(StorageAddonBase, '_get_file_tree') as mock_file_tree:
            mock_file_tree.return_value = FILE_TREE
            results = [stat_addon(addon, src_pk, dst_pk, user_pk) for addon in ['osfstorage', 'dropbox']]
        with mock.patch.object(celery, 'group') as mock_group:
            archive_node(results, src_pk, dst_pk, user_pk)
        archive_dropbox_signature = archive_addon.si(
            'dropbox',
            src_pk,
            dst_pk,
            user_pk,
            result
        )
        assert(mock_group.called_with(archive_dropbox_signature))

    def test_archive_node_fail(self):
        settings.MAX_ARCHIVE_SIZE = 100
        src_pk, dst_pk, user_pk = self.pks
        with mock.patch.object(StorageAddonBase, '_get_file_tree') as mock_file_tree:
            mock_file_tree.return_value = FILE_TREE
            results = [stat_addon(addon, src_pk, dst_pk, user_pk) for addon in ['osfstorage', 'dropbox']]
        with mock.patch('website.archiver.utils.handle_archive_fail') as mock_fail:
            archive_node(results, src_pk, dst_pk, user_pk)
        assert_equal(
            mock_fail.call_args_list[0][0][:-1],
            (
                ARCHIVE_SIZE_EXCEEDED,
                self.src,
                self.dst,
                self.user,
            )
        )
<commit_msg>Fix more failing archiver tests<commit_after>import json
import celery
from faker import Faker
import datetime
from modularodm import Q
import requests

import mock  # noqa
from mock import call
from nose.tools import *  # noqa PEP8 asserts
import httpretty

from scripts import cleanup_failed_registrations as scripts

from framework.auth import Auth
from framework.tasks import handlers

from website.archiver import (
    ARCHIVER_CHECKING,
    ARCHIVER_PENDING,
    ARCHIVER_SUCCESS,
    ARCHIVER_FAILURE,
    ARCHIVE_COPY_FAIL,
    ARCHIVE_SIZE_EXCEEDED,
)
from website.archiver import utils as archiver_utils
from website.app import *  # noqa
from website import archiver
from website.archiver import listeners
from website.archiver.tasks import *   # noqa

from website import mails
from website import settings
from website.util import waterbutler_url_for
from website.project.model import Node
from website.addons.base import StorageAddonBase

from tests import factories
from tests.base import OsfTestCase

fake = Faker()

FILE_TREE = {
    'path': '/',
    'name': '',
    'kind': 'folder',
    'children': [
        {
            'path': '/1234567',
            'name': 'Afile.file',
            'kind': 'file',
            'size': '128',
        },
        {
            'path': '/qwerty',
            'name': 'A Folder',
            'kind': 'folder',
            'children': [
                {
                    'path': '/qwerty/asdfgh',
                    'name': 'coolphoto.png',
                    'kind': 'file',
                    'size': '256',
                }
            ],
        }
    ],
}

class ArchiverTestCase(OsfTestCase):
    def setUp(self):
        super(ArchiverTestCase, self).setUp()
        handlers.celery_before_request()
        self.user = factories.UserFactory()
        self.auth = Auth(user=self.user)
        self.src = factories.NodeFactory(creator=self.user)
        self.src.add_addon('dropbox', auth=self.auth)
        self.dst = factories.RegistrationFactory(user=self.user, project=self.src, send_signals=False)
        self.stat_result = archiver_utils.aggregate_file_tree_metadata('dropbox', FILE_TREE, self.user)
        self.pks = (self.src._id, self.dst._id, self.user._id)

class TestStorageAddonBase(ArchiverTestCase):

    RESP_MAP = {
        '/': dict(data=FILE_TREE['children']),
        '/1234567': dict(data=FILE_TREE['children'][0]),
        '/qwerty': dict(data=FILE_TREE['children'][1]['children']),
        '/qwerty/asdfgh': dict(data=FILE_TREE['children'][1]['children'][0]),
    }

    @httpretty.activate
    def _test__get_file_tree(self, addon_short_name):
        requests_made = []
        def callback(request, uri, headers):
            path = request.querystring['path'][0]
            requests_made.append(path)
            return (200, headers, json.dumps(self.RESP_MAP[path]))

        for path in self.RESP_MAP.keys():
            url = waterbutler_url_for(
                'metadata',
                provider=addon_short_name,
                path=path,
                node=self.src,
                user=self.user,
                view_only=True,
            )
            httpretty.register_uri(httpretty.GET,
                                   url,
                                   body=callback,
                                   content_type='applcation/json')
        addon = self.src.get_or_add_addon(addon_short_name, auth=self.auth)
        root = {
            'path': '/',
            'name': '',
            'kind': 'folder',
        }
        file_tree = addon._get_file_tree(root, self.user)
        assert_equal(FILE_TREE, file_tree)
        assert_equal(requests_made, ['/', '/qwerty'])  # no requests made for files

    def _test_addon(self, addon_short_name):
        self._test__get_file_tree(addon_short_name)

    def test_addons(self):
        #  Test that each addon in settings.ADDONS_ARCHIVABLE if not addon == 'wiki' implements the StorageAddonBase interface
        for addon in [a for a in settings.ADDONS_ARCHIVABLE if not a == 'wiki']:
            self._test_addon(addon)

class TestArchiverTasks(ArchiverTestCase):

    @mock.patch('website.archiver.tasks.stat_node.delay')
    def test_archive(self, mock_stat):
        src_pk, dst_pk, user_pk = self.pks
        archive(src_pk, dst_pk, user_pk)
        mock_stat.assert_called_with(src_pk, dst_pk, user_pk)
        assert_true(self.dst.archiving)

    def test_stat_node(self):
        src_pk, dst_pk, user_pk = self.pks
        self.src.add_addon('box', auth=self.auth)  # has box, dropbox
        with mock.patch('celery.group') as mock_group:
            stat_node.apply(src_pk, dst_pk, user_pk)
        stat_dropbox_sig = stat_addon.si('dropbox', src_pk, dst_pk, user_pk)
        stat_box_sig = stat_addon.si('box', src_pk, dst_pk, user_pk)
        assert(mock_group.called_with(stat_dropbox_sig, stat_box_sig))

    def test_stat_addon(self):
        src_pk, dst_pk, user_pk = self.pks
        with mock.patch.object(StorageAddonBase, '_get_file_tree') as mock_file_tree:
            with mock.patch.object(archiver, 'AggregateStatResult') as MockStat:
                mock_file_tree.return_value = FILE_TREE
                res = stat_addon('dropbox', src_pk,  dst_pk, user_pk)
        assert_equal(self.dst.archived_providers['dropbox']['status'], ARCHIVER_CHECKING)
        src_dropbox = self.src.get_addon('dropbox')
        assert(MockStat.called_with(
            src_dropbox._id,
            'dropbox',
            targets=[archiver_utils.aggregate_file_tree_metadata(src_dropbox, FILE_TREE, self.user)]
        ))
        assert_equal(res.target_name, 'dropbox')

    def test_archive_node_pass(self):
        src_pk, dst_pk, user_pk = self.pks
        with mock.patch.object(StorageAddonBase, '_get_file_tree') as mock_file_tree:
            mock_file_tree.return_value = FILE_TREE
            results = [stat_addon(addon, src_pk, dst_pk, user_pk) for addon in ['osfstorage', 'dropbox']]
        with mock.patch.object(celery, 'group') as mock_group:
            archive_node(results, src_pk, dst_pk, user_pk)
        archive_dropbox_signature = archive_addon.si(
            'dropbox',
            src_pk,
            dst_pk,
            user_pk,
            results
        )
        assert(mock_group.called_with(archive_dropbox_signature))

    def test_archive_node_fail(self):
        settings.MAX_ARCHIVE_SIZE = 100
        src_pk, dst_pk, user_pk = self.pks
        with mock.patch.object(StorageAddonBase, '_get_file_tree') as mock_file_tree:
            mock_file_tree.return_value = FILE_TREE
            results = [stat_addon(addon, src_pk, dst_pk, user_pk) for addon in ['osfstorage', 'dropbox']]
        with mock.patch('website.archiver.utils.handle_archive_fail') as mock_fail:
            archive_node(results, src_pk, dst_pk, user_pk)
        assert_equal(
            mock_fail.call_args_list[0][0][:-1],
            (
                ARCHIVE_SIZE_EXCEEDED,
                self.src,
                self.dst,
                self.user,
            )
        )

{'fraction_non_alphanumeric': 0.10472279260780287, 'fraction_numerical': 0.004956454011187425, 'mean_word_length': 2.3903024483917426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6164264', 'n_tokens_mistral': 4528, 'n_tokens_neox': 4419, 'n_words': 842}