<commit_before>import logging
import logging.config
import signal

import click
from elasticsearch_dsl.connections import connections
from rdflib import Graph
import stomp

from pit.index import Thesis, indexable, ThesisResource
from pit.logging import BASE_CONFIG


@click.group()
def main():
    logging.config.dictConfig(BASE_CONFIG)


@main.command()
@click.option('--broker-host', default='localhost')
@click.option('--broker-port', default=61613)
@click.option('--index-host', default='localhost')
@click.option('--index-port', default=9200)
@click.option('--repo-host', default='localhost')
@click.option('--repo-port', default=80)
@click.option('--queue', default='/queue/fedora')
def run(broker_host, broker_port, index_host, index_port, repo_host,
        repo_port, queue):
    logger = logging.getLogger(__name__)
    es_conn = "{}:{}".format(index_host, index_port)
    connections.create_connection(hosts=[es_conn], timeout=20)
    logger.debug('Connected to Elasticsearch on {}'.format(es_conn))
    Thesis.init()
    conn = stomp.Connection([(broker_host, broker_port)])
    conn.set_listener('indexer', FedoraListener())
    conn.start()
    conn.connect(wait=True)
    logger.debug('Connected to ActiveMQ on {}:{}'.format(broker_host,
                                                         broker_port))
    conn.subscribe(queue, 1)
    while True:
        signal.pause()


class FedoraListener(stomp.ConnectionListener):
    def on_message(self, headers, message):
        logger = logging.getLogger(__name__)
        logger.debug('Message arrived: {}'.format(headers))
        if indexable(headers):
            logger.debug('Processing message {}'\
                .format(headers['message-id']))
            handle_message(message)


def handle_message(message):
    logger = logging.getLogger(__name__)
    t = ThesisResource(Graph().parse(data=message, format='json-ld'))
    Thesis(
        title=t.title,
        department=t.department,
        reviewer=t.reviewer,
        author=t.author,
        issue_date=t.issue_date,
        copyright_date=t.copyright_date,<commit_msg>Cleanly shutdown when the container is stopped<commit_after>import logging
import logging.config
import signal
import sys

import click
from elasticsearch_dsl.connections import connections
from rdflib import Graph
import stomp

from pit.index import Thesis, indexable, ThesisResource
from pit.logging import BASE_CONFIG


@click.group()
def main():
    logging.config.dictConfig(BASE_CONFIG)


@main.command()
@click.option('--broker-host', default='localhost')
@click.option('--broker-port', default=61613)
@click.option('--index-host', default='localhost')
@click.option('--index-port', default=9200)
@click.option('--repo-host', default='localhost')
@click.option('--repo-port', default=80)
@click.option('--queue', default='/queue/fedora')
def run(broker_host, broker_port, index_host, index_port, repo_host,
        repo_port, queue):
    logger = logging.getLogger(__name__)
    es_conn = "{}:{}".format(index_host, index_port)
    connections.create_connection(hosts=[es_conn], timeout=20)
    logger.debug('Connected to Elasticsearch on {}'.format(es_conn))
    Thesis.init()
    conn = stomp.Connection([(broker_host, broker_port)])
    conn.set_listener('indexer', FedoraListener())
    conn.start()
    conn.connect(wait=True)
    logger.debug('Connected to ActiveMQ on {}:{}'.format(broker_host,
                                                         broker_port))
    conn.subscribe(queue, 1)

    def shutdown(signum, stack):
        conn.disconnect()
        logger.debug('Disconnected from ActiveMQ')
        sys.exit(0)

    signal.signal(signal.SIGTERM, shutdown)

    while True:
        signal.pause()


class FedoraListener(stomp.ConnectionListener):
    def on_message(self, headers, message):
        logger = logging.getLogger(__name__)
        if indexable(headers):
            logger.debug('Processing message {}'\
                .format(headers['message-id']))
            handle_message(message)


def handle_message(message):
    logger = logging.getLogger(__name__)
    t = ThesisResource(Graph().parse(data=message, format='json-ld'))
    Thesis(
        title=t.title,
        department=t.department,
        reviewer=t.reviewer,
        author=t.author,
        issue_date=t.issue_date,
        copyright_date=t.copyright_date,
{'fraction_non_alphanumeric': 0.13337908945321436, 'fraction_numerical': 0.0066346373827499425, 'mean_word_length': 3.5828092243186584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4753626', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1284, 'n_words': 248}