<commit_before>import ddt
import hashlib
import json
import falcon
import msgpack
import os
from random import randrange
import uuid
from mock import patch
from deuce import conf
from deuce.util.misc import set_qs, relative_uri
from six.moves.urllib.parse import urlparse, parse_qs

from deuce.tests import ControllerTest


@ddt.ddt
class TestBlocksController(ControllerTest):

    def setUp(self):
        super(TestBlocksController, self).setUp()

        # Create a vault for us to work with

        vault_name = self.create_vault_id()
        self._vault_path = '/v1.0/vaults/{0}'.format(vault_name)
        self._blocks_path = '{0}/blocks'.format(self._vault_path)

        self._files_path = self._vault_path + '/files'
        self._hdrs = {"x-project-id": self.create_project_id()}

        response = self.simulate_put(self._vault_path,
                                     headers=self._hdrs)

        self.block_list = []
        self.total_block_num = 0

    def test_no_block_state(self):
        # Try listing the blocks. There should be none
        response = self.simulate_get(self._blocks_path, headers=self._hdrs)
        self.assertEqual(response[0].decode(), json.dumps([]))

    def test_get_all_with_trailing_slash(self):
        path = self.get_block_path('')

        response = self.simulate_get(path, headers=self._hdrs)
        self.assertEqual(self.srmock.status, falcon.HTTP_200)

    def test_get_all_invalid_vault_id(self):
        path = '/v1.0/vaults/{0}/blocks'.format('bad_vault_id')
        response = self.simulate_get(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_get_invalid_block_id(self):
        path = self.get_block_path('invalid_block_id')
        response = self.simulate_get(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_invalid_block_id(self):
        path = self.get_block_path('invalid_block_id')
        response = self.simulate_get(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_non_existent_block_id(self):
        path = self.get_block_path(self.calc_sha1(b'mock'))
        response = self.simulate_head(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_inconsistent_metadata_block_id(self):
        from deuce.model import Vault
        with patch.object(Vault, '_storage_has_block', return_value=False):
            block_list = self.helper_create_blocks(1, async=True)
            path = self.get_block_path(block_list[0])
            self.simulate_head(path, headers=self._hdrs)
            self.assertEqual(self.srmock.status, falcon.HTTP_502)

    def test_head_block_nonexistent_vault(self):
        self.simulate_head('/v1.0/vaults/mock/blocks/'
                           + self.calc_sha1(b'mock'), headers=self._hdrs)
        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_block(self):
        block_list = self.helper_create_blocks(1, async=True)
        path = self.get_block_path(block_list[0])
        self.simulate_head(path, headers=self._hdrs)
        self.assertEqual(self.srmock.status, falcon.HTTP_204)
        self.assertIn('x-block-reference-count', str(self.srmock.headers))
        self.assertIn('x-ref-modified', str(self.srmock.headers))
        self.assertIn('x-storage-id', str(self.srmock.headers))
        self.assertTrue(uuid.UUID(self.srmock.headers_dict['x-storage-id']))

    def test_put_invalid_block_id(self):
        path = self.get_block_path('invalid_block_id')

        response = self.simulate_put(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_400)
<commit_msg>Enhancement: More complete testing<commit_after>import ddt
import hashlib
import json
import falcon
import msgpack
import os
from random import randrange
import uuid
from mock import patch
from deuce import conf
from deuce.util.misc import set_qs, relative_uri
from six.moves.urllib.parse import urlparse, parse_qs

from deuce.tests import ControllerTest


@ddt.ddt
class TestBlocksController(ControllerTest):

    def setUp(self):
        super(TestBlocksController, self).setUp()

        # Create a vault for us to work with

        vault_name = self.create_vault_id()
        self._vault_path = '/v1.0/vaults/{0}'.format(vault_name)
        self._blocks_path = '{0}/blocks'.format(self._vault_path)

        self._files_path = self._vault_path + '/files'
        self._hdrs = {"x-project-id": self.create_project_id()}

        response = self.simulate_put(self._vault_path,
                                     headers=self._hdrs)

        self.block_list = []
        self.total_block_num = 0

    def test_no_block_state(self):
        # Try listing the blocks. There should be none
        response = self.simulate_get(self._blocks_path, headers=self._hdrs)
        self.assertEqual(response[0].decode(), json.dumps([]))

    def test_get_all_with_trailing_slash(self):
        path = self.get_block_path('')

        response = self.simulate_get(path, headers=self._hdrs)
        self.assertEqual(self.srmock.status, falcon.HTTP_200)

    def test_get_all_invalid_vault_id(self):
        path = '/v1.0/vaults/{0}/blocks'.format('bad_vault_id')
        response = self.simulate_get(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_get_invalid_block_id(self):
        path = self.get_block_path('invalid_block_id')
        response = self.simulate_get(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_invalid_block_id(self):
        path = self.get_block_path('invalid_block_id')
        response = self.simulate_get(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_non_existent_block_id(self):
        path = self.get_block_path(self.calc_sha1(b'mock'))
        response = self.simulate_head(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_inconsistent_metadata_block_id(self):
        from deuce.model import Vault
        with patch.object(Vault, '_storage_has_block', return_value=False):
            block_list = self.helper_create_blocks(1, async=True)
            path = self.get_block_path(block_list[0])
            self.simulate_head(path, headers=self._hdrs)
            self.assertEqual(self.srmock.status, falcon.HTTP_502)

    def test_head_block_nonexistent_vault(self):
        self.simulate_head('/v1.0/vaults/mock/blocks/'
                           + self.calc_sha1(b'mock'), headers=self._hdrs)
        self.assertEqual(self.srmock.status, falcon.HTTP_404)

    def test_head_block(self):
        block_list = self.helper_create_blocks(1, async=True)
        path = self.get_block_path(block_list[0])
        self.simulate_head(path, headers=self._hdrs)
        self.assertEqual(self.srmock.status, falcon.HTTP_204)
        self.assertIn('x-block-reference-count', str(self.srmock.headers))
        self.assertIn('x-ref-modified', str(self.srmock.headers))
        self.assertIn('x-storage-id', str(self.srmock.headers))
        self.assertTrue(uuid.UUID(self.srmock.headers_dict['x-storage-id']))
        self.assertIn('x-block-id', str(self.srmock.headers))
        self.assertEqual(block_list[0], self.srmock.headers_dict['x-block-id'])

    def test_put_invalid_block_id(self):
        path = self.get_block_path('invalid_block_id')

        response = self.simulate_put(path, headers=self._hdrs)

        self.assertEqual(self.srmock.status, falcon.HTTP_400)

{'fraction_non_alphanumeric': 0.11047495120364346, 'fraction_numerical': 0.01158100195185426, 'mean_word_length': 3.279510022271715, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6169990', 'n_tokens_mistral': 2597, 'n_tokens_neox': 2511, 'n_words': 379}