# Path: processing/load_body.py
# SENTRY_DSN = os.environ.get('SENTRY_DSN', None)
# LOGGING_LEVEL = os.environ.get('LOGGING_LEVEL', 'DEBUG')
# MONGODB_HOST = os.environ.get('MONGODB_HOST', None)
# LOGGING = {
#     'version': 1,
#     'disable_existing_loggers': True,
# 
#     'formatters': {
#         'console': {
#             'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
#             'datefmt': '%H:%M:%S',
#             },
#         },
#     'handlers': {
#         'console': {
#             'level': LOGGING_LEVEL,
#             'class': 'logging.StreamHandler',
#             'formatter': 'console'
#             }
#         },
#     'loggers': {
#         '': {
#             'handlers': ['console'],
#             'level': LOGGING_LEVEL,
#             'propagate': False,
#             },
#         'processing.load_body': {
#             'level': LOGGING_LEVEL,
#             'propagate': True,
#         },
#     }
# }
# FROM = datetime.now() - timedelta(days=15)
# FROM = FROM.isoformat()[:10]
# BODY_REGEX = re.compile(r'<div .*class="index,(?P<language>.*?)">(?P<body>.*)</div>')
# REMOVE_LINKS_REGEX = re.compile(r'\[.<a href="javascript\:void\(0\);".*?>Links</a>.\]', re.IGNORECASE)
# def collections_acronym(articlemeta_db):
# def collection_info(articlemeta_db, collection):
# def load_documents_pids(articlemeta_db, pids, collection):
# def load_documents_collection(articlemeta_db, collection, all_records=False):
# def do_request(url, json=True):
# def scrap_body(data, language):
# def add_bodies(articlemeta_db, documents, collection):
# def run(articlemeta_db, collections, pids=None, all_records=False):
# def main():


import unittest
import os
import codecs
        import re
        import re
        import re
from processing import load_body
        result = load_body.scrap_body(data.encode('utf-8'), 'pt')

        # Text on the begining of the document
        self.assertTrue(u'A TRIBUTAÇÃO NA PRODUÇÃO DE CARVÃO VEGETAL' in result)
        # Text on the end of the document
        self.assertTrue(u'Recebido: 03 de Fevereiro de 2012; Aceito: 14 de Abril de 2014' in result)

    def test_body_sample_4(self):
        data = codecs.open(os.path.dirname(__file__)+'/fixtures/body_sample_4.html', 'r', encoding='utf-8').read()

        result = load_body.scrap_body(data.encode('utf-8'), 'pt')

        # Text on the begining of the document
        self.assertTrue(u'Aquarelas de um Brasil' in result)
        # Text on the end of the document
        self.assertTrue(u'São Paulo, Companhia das Letras.' in result)

    def test_body_sample_5(self):
        data = codecs.open(os.path.dirname(__file__)+'/fixtures/body_sample_5.html', 'r', encoding='utf-8').read()

        result = load_body.scrap_body(data.encode('utf-8'), 'pt')

        # Text on the begining of the document
        self.assertTrue(u'Molestia de Carlos Chagas' in result)
        # Text on the end of the document
        self.assertTrue(u'Full text available only in PDF format' in result)

    def test_body_sample_6(self):
        data = codecs.open(os.path.dirname(__file__)+'/fixtures/body_sample_6.html', 'r', encoding='utf-8').read()


# Path: processing/load_body.py
# SENTRY_DSN = os.environ.get('SENTRY_DSN', None)
# LOGGING_LEVEL = os.environ.get('LOGGING_LEVEL', 'DEBUG')
# MONGODB_HOST = os.environ.get('MONGODB_HOST', None)
# LOGGING = {
#     'version': 1,
#     'disable_existing_loggers': True,
# 
#     'formatters': {
#         'console': {
#             'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
#             'datefmt': '%H:%M:%S',
#             },
#         },
#     'handlers': {
#         'console': {
#             'level': LOGGING_LEVEL,
#             'class': 'logging.StreamHandler',
#             'formatter': 'console'
#             }
#         },
#     'loggers': {
#         '': {
#             'handlers': ['console'],
#             'level': LOGGING_LEVEL,
#             'propagate': False,
#             },
#         'processing.load_body': {
#             'level': LOGGING_LEVEL,
#             'propagate': True,
#         },
#     }
# }
# FROM = datetime.now() - timedelta(days=15)
# FROM = FROM.isoformat()[:10]
# BODY_REGEX = re.compile(r'<div .*class="index,(?P<language>.*?)">(?P<body>.*)</div>')
# REMOVE_LINKS_REGEX = re.compile(r'\[.<a href="javascript\:void\(0\);".*?>Links</a>.\]', re.IGNORECASE)
# def collections_acronym(articlemeta_db):
# def collection_info(articlemeta_db, collection):
# def load_documents_pids(articlemeta_db, pids, collection):
# def load_documents_collection(articlemeta_db, collection, all_records=False):
# def do_request(url, json=True):
# def scrap_body(data, language):
# def add_bodies(articlemeta_db, documents, collection):
# def run(articlemeta_db, collections, pids=None, all_records=False):
# def main():


# Path: tests/test_load_body.py
import unittest
import os
import codecs
        import re
        import re
        import re
from processing import load_body

        result = load_body.scrap_body(data.encode('utf-8'), 'pt')

        # Text on the begining of the document
        self.assertTrue(u'A TRIBUTAÇÃO NA PRODUÇÃO DE CARVÃO VEGETAL' in result)
        # Text on the end of the document
        self.assertTrue(u'Recebido: 03 de Fevereiro de 2012; Aceito: 14 de Abril de 2014' in result)

    def test_body_sample_4(self):
        data = codecs.open(os.path.dirname(__file__)+'/fixtures/body_sample_4.html', 'r', encoding='utf-8').read()

        result = load_body.scrap_body(data.encode('utf-8'), 'pt')

        # Text on the begining of the document
        self.assertTrue(u'Aquarelas de um Brasil' in result)
        # Text on the end of the document
        self.assertTrue(u'São Paulo, Companhia das Letras.' in result)

    def test_body_sample_5(self):
        data = codecs.open(os.path.dirname(__file__)+'/fixtures/body_sample_5.html', 'r', encoding='utf-8').read()

        result = load_body.scrap_body(data.encode('utf-8'), 'pt')

        # Text on the begining of the document
        self.assertTrue(u'Molestia de Carlos Chagas' in result)
        # Text on the end of the document
        self.assertTrue(u'Full text available only in PDF format' in result)

    def test_body_sample_6(self):
        data = codecs.open(os.path.dirname(__file__)+'/fixtures/body_sample_6.html', 'r', encoding='utf-8').read()


        result = load_body.scrap_body(data.encode('utf-8'), 'pt')