<commit_before># coding: utf-8

__author__ = 'jociel'

import logging
import ConfigParser
import solr
from harvest import harvest
import solr_util
import time


def delete_article_entry(code):
    print(code)
    solr_conn.delete(code=code)


def add_update_article_entry(code, document, action, indexed_date):
    args = solr_util.get_solr_args_from_article(document, indexed_date)
    while True:
        try:
            solr_conn.add(**args)
            break
        except Exception as ex:
            logging.error(
                "An error has occurred trying to access Solr. Arguments passed to Solr and the traceback are below:")
            logging.error(args)
            logging.exception(ex)
            logging.error("Sleeping for 1 minute to try again...")
            time.sleep(60)


if __name__ == '__main__':
    FORMAT = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.INFO, format=FORMAT)

    config = ConfigParser.ConfigParser()
    config.read('harvest.cfg')

    solr_uri = config.get("harvest", "solr_uri")
    solr_conn = solr.SolrConnection(solr_uri)

    harvest(config.get("harvest", "article_meta_uri"),
            "article",
            config.get("harvest", "data_source_name"),<commit_msg>Deixando processamento resiliente, em casos de problemas para produzir o registro de carga no indice do Solr<commit_after># coding: utf-8

__author__ = 'jociel'

import logging
import ConfigParser
import solr
from harvest import harvest
import solr_util
import time


def delete_article_entry(code):
    print(code)
    solr_conn.delete(code=code)


def add_update_article_entry(code, document, action, indexed_date):
    try:
        args = solr_util.get_solr_args_from_article(document, indexed_date)
    except Exception as ex:
        logging.exception(ex)

    while True:
        try:
            solr_conn.add(**args)
            break
        except Exception as ex:
            logging.error(
                "An error has occurred trying to access Solr. Arguments passed to Solr and the traceback are below:")
            logging.error(args)
            logging.exception(ex)
            logging.error("Sleeping for 1 minute to try again...")
            time.sleep(60)


if __name__ == '__main__':
    FORMAT = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.INFO, format=FORMAT)

    config = ConfigParser.ConfigParser()
    config.read('harvest.cfg')

    solr_uri = config.get("harvest", "solr_uri")
    solr_conn = solr.SolrConnection(solr_uri)

    harvest(config.get("harvest", "article_meta_uri"),
            "article",
            config.get("harvest", "data_source_name"),
{'fraction_non_alphanumeric': 0.09860745201354912, 'fraction_numerical': 0.004516371847948814, 'mean_word_length': 2.9612518628912072, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7373171', 'n_tokens_mistral': 837, 'n_tokens_neox': 802, 'n_words': 199}