<commit_before>import yaml
import newspaper
from newspaper import news_pool


def get_config():
    with open('/options/config.yml') as file:
        config = yaml.safe_load(file)
    keys = config.keys()
    if not 'threads' in keys:
        config['threads'] = 2
    if not 'source' in keys:
        raise Exception('"source" not defined in config.yml')
    if not 'words' in keys:
        raise Exception('"words" not defined in config.yml')
    return config


def main():
    config = get_config()
    target_words_set = set([i.lower() for i in config['words']])
    paper = newspaper.build(config['source'])
    news_pool.set([paper], threads_per_source=config['threads'])
    news_pool.join()

    for a in paper.articles:
        a.parse()
        a.nlp()
        if target_words_set < set(a.keywords):
            print(a.title)
            print(a.keywords)
            # TODO: hand article to queue for processing


if __name__ == '__main__':
    main()<commit_msg>Build paper and hand articles to queue<commit_after>import yaml
import newspaper
from rabbit.publisher import Publisher


def main():
    with open('/options/config.yml') as file:
        config = yaml.safe_load(file)
    paper = newspaper.build(config['source'])
    publisher = Publisher(
        rabbit_url=config['rabbit_url'],
        publish_interval=1,
        article_urls=paper.articles)
    publisher.run()


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.11731843575418995, 'fraction_numerical': 0.0013966480446927375, 'mean_word_length': 2.9368131868131866, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '560827', 'n_tokens_mistral': 441, 'n_tokens_neox': 421, 'n_words': 119}