# Path: scrapi/processing/elasticsearch.py
# class ElasticsearchProcessor(BaseProcessor):
#     NAME = 'elasticsearch'
# 
#     manager = DatabaseManager()
# 
#     def documents(self, *sources):
#         raise NotImplementedError
# 
#     def process_normalized(self, raw_doc, normalized, index=None):
#         index = index or settings.ELASTIC_INDEX
#         data = {
#             key: value for key, value in normalized.attributes.items()
#             if key in (settings.FRONTEND_KEYS or normalized.attributes.keys())
#         }
#         data['providerUpdatedDateTime'] = self.version(raw_doc, normalized)
# 
#         # check shareProperties for status and delete if it is deleted
#         if data['shareProperties'].get('status', None) == 'deleted':
#             self.manager.es.delete(
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#                 ignore=[404]
#             )
#         else:
#             self.manager.es.index(
#                 body=data,
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#             )
#         self.process_normalized_v1(raw_doc, normalized, data['providerUpdatedDateTime'])
# 
#     def version(self, raw, normalized):
#         try:
#             old_doc = self.manager.es.get_source(
#                 index=settings.ELASTIC_INDEX,
#                 doc_type=raw['source'],
#                 id=raw['docID']
#             )
#         except NotFoundError:  # pragma: no cover
#             # Normally I don't like exception-driven logic,
#             # but this was the best way to handle missing
#             # types, indices and documents together
#             date = normalized['providerUpdatedDateTime']
#         else:
#             date = old_doc.get('providerUpdatedDateTime') or normalized['providerUpdatedDateTime']
# 
#         return date
# 
#     def process_normalized_v1(self, raw_doc, normalized, date):
#         index = 'share_v1'
#         transformer = PreserveOldSchema()
#         data = transformer.transform(normalized.attributes)
#         data['providerUpdatedDateTime'] = date
#         # check shareProperties for status and delete if it is deleted
#         if normalized.attributes['shareProperties'].get('status', None) == 'deleted':
#             self.manager.es.delete(
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#                 ignore=[404]
#             )
#         else:
#             self.manager.es.index(
#                 body=data,
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#             )
# 
#     def get(self, docID, index, source):
#         try:
#             results = self.manager.es.get_source(
#                 index=index,
#                 doc_type=source,
#                 id=docID
#             )
#         except NotFoundError:
#             return None
# 
#         return DocumentTuple(None, NormalizedDocument(results, validate=False, clean=False))


import pytest
from . import utils
from scrapi.linter.document import NormalizedDocument, RawDocument
from scrapi.processing.elasticsearch import ElasticsearchProcessor


test_db = ElasticsearchProcessor()
es = test_db.manager.es

RAW = RawDocument(utils.RAW_DOC)
NORMALIZED = NormalizedDocument(utils.RECORD)


@pytest.mark.elasticsearch
def test_process_normalized():
    test_db.process_normalized(RAW, NORMALIZED, index='test')

    results = es.search(index='test', doc_type=RAW['source'])
    assert (len(results['hits']['hits']) == 1)


@pytest.mark.elasticsearch
def test_versions():

# Path: scrapi/processing/elasticsearch.py
# class ElasticsearchProcessor(BaseProcessor):
#     NAME = 'elasticsearch'
# 
#     manager = DatabaseManager()
# 
#     def documents(self, *sources):
#         raise NotImplementedError
# 
#     def process_normalized(self, raw_doc, normalized, index=None):
#         index = index or settings.ELASTIC_INDEX
#         data = {
#             key: value for key, value in normalized.attributes.items()
#             if key in (settings.FRONTEND_KEYS or normalized.attributes.keys())
#         }
#         data['providerUpdatedDateTime'] = self.version(raw_doc, normalized)
# 
#         # check shareProperties for status and delete if it is deleted
#         if data['shareProperties'].get('status', None) == 'deleted':
#             self.manager.es.delete(
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#                 ignore=[404]
#             )
#         else:
#             self.manager.es.index(
#                 body=data,
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#             )
#         self.process_normalized_v1(raw_doc, normalized, data['providerUpdatedDateTime'])
# 
#     def version(self, raw, normalized):
#         try:
#             old_doc = self.manager.es.get_source(
#                 index=settings.ELASTIC_INDEX,
#                 doc_type=raw['source'],
#                 id=raw['docID']
#             )
#         except NotFoundError:  # pragma: no cover
#             # Normally I don't like exception-driven logic,
#             # but this was the best way to handle missing
#             # types, indices and documents together
#             date = normalized['providerUpdatedDateTime']
#         else:
#             date = old_doc.get('providerUpdatedDateTime') or normalized['providerUpdatedDateTime']
# 
#         return date
# 
#     def process_normalized_v1(self, raw_doc, normalized, date):
#         index = 'share_v1'
#         transformer = PreserveOldSchema()
#         data = transformer.transform(normalized.attributes)
#         data['providerUpdatedDateTime'] = date
#         # check shareProperties for status and delete if it is deleted
#         if normalized.attributes['shareProperties'].get('status', None) == 'deleted':
#             self.manager.es.delete(
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#                 ignore=[404]
#             )
#         else:
#             self.manager.es.index(
#                 body=data,
#                 refresh=True,
#                 index=index,
#                 doc_type=raw_doc['source'],
#                 id=raw_doc['docID'],
#             )
# 
#     def get(self, docID, index, source):
#         try:
#             results = self.manager.es.get_source(
#                 index=index,
#                 doc_type=source,
#                 id=docID
#             )
#         except NotFoundError:
#             return None
# 
#         return DocumentTuple(None, NormalizedDocument(results, validate=False, clean=False))


# Path: tests/test_elasticsearch_processor.py
import pytest
from . import utils
from scrapi.linter.document import NormalizedDocument, RawDocument
from scrapi.processing.elasticsearch import ElasticsearchProcessor



test_db = ElasticsearchProcessor()
es = test_db.manager.es

RAW = RawDocument(utils.RAW_DOC)
NORMALIZED = NormalizedDocument(utils.RECORD)


@pytest.mark.elasticsearch
def test_process_normalized():
    test_db.process_normalized(RAW, NORMALIZED, index='test')

    results = es.search(index='test', doc_type=RAW['source'])
    assert (len(results['hits']['hits']) == 1)


@pytest.mark.elasticsearch
def test_versions():

    NORMALIZED['source'] = RAW['source']