<commit_before>'''
Module for testing the Elasticsearch datastore.
'''
import os
import sys
import mock
import unittest

from elasticsearch import Elasticsearch
from elasticsearch.client import IndicesClient, IngestClient

CWD = os.path.dirname(os.path.abspath(__file__))
MS_WD = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Allow import of elasticsearch_storage
if os.path.join(MS_WD, 'storage') not in sys.path:
    sys.path.insert(0, os.path.join(MS_WD, 'storage'))
# Use multiscanner in ../
sys.path.insert(0, os.path.dirname(CWD))

# import elasticsearch_storage
from elasticsearch_storage import ElasticSearchStorage

TEST_MS_OUTPUT = {'test.txt': {'SHA1': '02bed644797a7adb7d9e3fe8246cc3e1caed0dfe', 'MD5': 'd74129f99f532292de5db9a90ec9d424', 'libmagic': 'ASCII text, with very long lines, with no line terminators', 'ssdeep': '6:BLWw/ELmRCp8o7cu5eul3tkxZBBCGAAIwLE/mUz9kLTCDFM1K7NBVn4+MUq08:4w/ELmR48oJh1exX8G7TW+wM1uFwp', 'SHA256': '03a634eb98ec54d5f7a3c964a82635359611d84dd4ba48e860e6d4817d4ca2a6', 'Metadata': {}, "Scan Time": "2017-09-26T16:48:05.395004"}}    # noqa: E501
TEST_ID = '03a634eb98ec54d5f7a3c964a82635359611d84dd4ba48e860e6d4817d4ca2a6'
TEST_NOTE_ID = 'eba3d3de-1a7e-4018-8fb3-a4635b4b7ab1'
TEST_TS = "2017-09-26T16:48:05.395004"


class TestES(unittest.TestCase):
    @mock.patch.object(IndicesClient, 'exists_template')
    @mock.patch.object(IngestClient, 'get_pipeline')
    def setUp(self, mock_pipe, mock_exists):
        mock_pipe.return_value = True
        mock_exists.return_value = True
        self.handler = ElasticSearchStorage(config=ElasticSearchStorage.DEFAULTCONF)
        self.handler.setup()

    @mock.patch.object(Elasticsearch, 'index')
    @mock.patch('elasticsearch_storage.helpers')
    def test_store(self, mock_helpers, mock_index):
        mock_helpers.bulk.return_value = (1, [])
        resp = self.handler.store(TEST_MS_OUTPUT)

        args, kwargs = mock_helpers.bulk.call_args_list[0]
        sample_args = args[1][0]
        self.assertEqual(sample_args['pipeline'], 'dedot')
        self.assertEqual(sample_args['_id'], TEST_ID)
        self.assertEqual(sample_args['_source']['SHA256'], TEST_ID)
        self.assertEqual(sample_args['_source']['tags'], [])

        report_args, report_kwargs = mock_index.call_args_list[0]
        self.assertEqual(report_kwargs['parent'], TEST_ID)
        self.assertEqual(
            report_kwargs['body']['libmagic'],
            'ASCII text, with very long lines, with no line terminators')
        self.assertEqual(report_kwargs['pipeline'], 'dedot')

        self.assertIn(TEST_ID, resp)

    @mock.patch.object(Elasticsearch, 'search')
    @mock.patch.object(Elasticsearch, 'get')
    def test_get_report(self, mock_get, mock_search):
        self.handler.get_report(TEST_ID, TEST_TS)

        args, kwargs = mock_search.call_args_list[0]
        self.assertEqual(kwargs['index'], ElasticSearchStorage.DEFAULTCONF['index'])
        self.assertEqual(kwargs['body']['query']['bool']['must'][0]['has_parent']['query']['term']['_id'], TEST_ID)
        self.assertEqual(kwargs['body']['query']['bool']['must'][1]['term']['Scan Time'], TEST_TS)
        self.assertEqual(kwargs['doc_type'], ElasticSearchStorage.DEFAULTCONF['doc_type'])

        mock_get.assert_any_call(index=ElasticSearchStorage.DEFAULTCONF['index'], id=TEST_ID, doc_type='sample')

    @mock.patch('elasticsearch_storage.helpers')
    def test_search(self, mock_helpers):
        mock_helpers.scan.return_value = [{'sort': [417], '_type': 'report', '_routing': TEST_ID, '_index': 'multiscanner_reports', '_score': None, '_source': {'libmagic': 'ASCII text, with very long lines, with no line terminators', 'filename': 'test.txt'}, '_parent': TEST_ID, '_id': TEST_ID}]     # noqa: E501
        resp = self.handler.search('test')

        args, kwargs = mock_helpers.scan.call_args_list[0]
        query = kwargs['query']
        self.assertEqual(query['query']['query_string']['query'], '*test*')<commit_msg>Update ES test to new scan time location<commit_after>'''
Module for testing the Elasticsearch datastore.
'''
import os
import sys
import mock
import unittest

from elasticsearch import Elasticsearch
from elasticsearch.client import IndicesClient, IngestClient

CWD = os.path.dirname(os.path.abspath(__file__))
MS_WD = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Allow import of elasticsearch_storage
if os.path.join(MS_WD, 'storage') not in sys.path:
    sys.path.insert(0, os.path.join(MS_WD, 'storage'))
# Use multiscanner in ../
sys.path.insert(0, os.path.dirname(CWD))

# import elasticsearch_storage
from elasticsearch_storage import ElasticSearchStorage

TEST_MS_OUTPUT = {'test.txt': {'SHA1': '02bed644797a7adb7d9e3fe8246cc3e1caed0dfe', 'MD5': 'd74129f99f532292de5db9a90ec9d424', 'libmagic': 'ASCII text, with very long lines, with no line terminators', 'ssdeep': '6:BLWw/ELmRCp8o7cu5eul3tkxZBBCGAAIwLE/mUz9kLTCDFM1K7NBVn4+MUq08:4w/ELmR48oJh1exX8G7TW+wM1uFwp', 'SHA256': '03a634eb98ec54d5f7a3c964a82635359611d84dd4ba48e860e6d4817d4ca2a6', 'Metadata': {}, "Scan Time": "2017-09-26T16:48:05.395004"}}    # noqa: E501
TEST_ID = '03a634eb98ec54d5f7a3c964a82635359611d84dd4ba48e860e6d4817d4ca2a6'
TEST_NOTE_ID = 'eba3d3de-1a7e-4018-8fb3-a4635b4b7ab1'
TEST_TS = "2017-09-26T16:48:05.395004"


class TestES(unittest.TestCase):
    @mock.patch.object(IndicesClient, 'exists_template')
    @mock.patch.object(IngestClient, 'get_pipeline')
    def setUp(self, mock_pipe, mock_exists):
        mock_pipe.return_value = True
        mock_exists.return_value = True
        self.handler = ElasticSearchStorage(config=ElasticSearchStorage.DEFAULTCONF)
        self.handler.setup()

    @mock.patch.object(Elasticsearch, 'index')
    @mock.patch('elasticsearch_storage.helpers')
    def test_store(self, mock_helpers, mock_index):
        mock_helpers.bulk.return_value = (1, [])
        resp = self.handler.store(TEST_MS_OUTPUT)

        args, kwargs = mock_helpers.bulk.call_args_list[0]
        sample_args = args[1][0]
        self.assertEqual(sample_args['pipeline'], 'dedot')
        self.assertEqual(sample_args['_id'], TEST_ID)
        self.assertEqual(sample_args['_source']['SHA256'], TEST_ID)
        self.assertEqual(sample_args['_source']['tags'], [])

        report_args, report_kwargs = mock_index.call_args_list[0]
        self.assertEqual(report_kwargs['parent'], TEST_ID)
        self.assertEqual(
            report_kwargs['body']['libmagic'],
            'ASCII text, with very long lines, with no line terminators')
        self.assertEqual(report_kwargs['pipeline'], 'dedot')

        self.assertIn(TEST_ID, resp)

    @mock.patch.object(Elasticsearch, 'search')
    @mock.patch.object(Elasticsearch, 'get')
    def test_get_report(self, mock_get, mock_search):
        self.handler.get_report(TEST_ID, TEST_TS)

        args, kwargs = mock_search.call_args_list[0]
        self.assertEqual(kwargs['index'], ElasticSearchStorage.DEFAULTCONF['index'])
        self.assertEqual(kwargs['body']['query']['bool']['must'][0]['has_parent']['query']['term']['_id'], TEST_ID)
        self.assertEqual(kwargs['body']['query']['bool']['must'][1]['term']['Scan Metadata.Scan Time'], TEST_TS)
        self.assertEqual(kwargs['doc_type'], ElasticSearchStorage.DEFAULTCONF['doc_type'])

        mock_get.assert_any_call(index=ElasticSearchStorage.DEFAULTCONF['index'], id=TEST_ID, doc_type='sample')

    @mock.patch('elasticsearch_storage.helpers')
    def test_search(self, mock_helpers):
        mock_helpers.scan.return_value = [{'sort': [417], '_type': 'report', '_routing': TEST_ID, '_index': 'multiscanner_reports', '_score': None, '_source': {'libmagic': 'ASCII text, with very long lines, with no line terminators', 'filename': 'test.txt'}, '_parent': TEST_ID, '_id': TEST_ID}]     # noqa: E501
        resp = self.handler.search('test')

        args, kwargs = mock_helpers.scan.call_args_list[0]
        query = kwargs['query']
        self.assertEqual(query['query']['query_string']['query'], '*test*')
{'fraction_non_alphanumeric': 0.14339105518873801, 'fraction_numerical': 0.056559112993646445, 'mean_word_length': 4.991044776119403, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3073675', 'n_tokens_mistral': 2970, 'n_tokens_neox': 2797, 'n_words': 437}