<commit_before>#Internal imports
from core.plugin_handler import subscribe
#External imports
import newspaper
#Builtin imports
import logging
import threading
import time

log = logging.getLogger()

def is_news(event):
    '''Determine whether to read the news'''
    event_words = [token.orth_.lower() for token in event["doc"]]
    return "news" in event_words

@subscribe({"name": "news", "check": is_news})
def news_reader(event):
    '''Use the excellent newspaper module to fetch the news from the readers favorite site'''
    response = {"type": "success", "text": None, "data": {}}
    db = event['db']
    event_user = event['username']
    user_table = db['users'].find_one(username=event_user)
    user_news_site = user_table["news_site"]
    news_table = db["news"]
    cached_sites = [site.values()[0] for site in db.query("SELECT site from `news`")]
    log.debug("Cached sites are {0}".format(cached_sites))
    if user_news_site in cached_sites:
        site_row = news_table.find_one(site=user_news_site)
        site_time = site_row["time"]
        if time.time()<site_time+43200:
            log.info("Using cached news for site {0}".format(user_news_site))
            news_str = site_row["news_str"]
            return news_str
    log.info("Parsing news site {0} for user {1}".format(user_news_site, event_user))
    site_object = newspaper.build(user_news_site)
    log.debug("Finished building newspaper object")<commit_msg>Adjust news plugin to new framework<commit_after>#Internal imports
from core.plugin_handler import subscribe
import tools
#External imports
import newspaper
#Builtin imports
import logging
import threading
import time

log = logging.getLogger()

def is_news(event):
    '''Determine whether to read the news'''
    event_words = [token.orth_.lower() for token in event["doc"]]
    return "news" in event_words

@subscribe({"name": "news", "check": is_news})
def news_reader(event):
    '''Use the excellent newspaper module to fetch the news from the readers favorite site'''
    response = {"type": "success", "text": None, "data": {}}
    db = event['db']
    event_user = event['username']
    user_table = db['users'].find_one(username=event_user)
    user_news_site = user_table["news_site"]
    news_table = db["news"]
    cached_sites = [site.values()[0] for site in db.query("SELECT site from `news`")]
    log.debug("Cached sites are {0}".format(cached_sites))
    if user_news_site in cached_sites:
        site_row = news_table.find_one(site=user_news_site)
        site_time = site_row["time"]
        if time.time()<site_time+43200:
            log.info("Using cached news for site {0}".format(user_news_site))
            news_str = site_row["news_str"]
            response["text"] = news_str
    log.info("Parsing news site {0} for user {1}".format(user_news_site, event_user))
    site_object = newspaper.build(user_news_site)
    log.debug("Finished building newspaper object")
{'fraction_non_alphanumeric': 0.12538226299694188, 'fraction_numerical': 0.0067957866123003734, 'mean_word_length': 4.210619469026549, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5962337', 'n_tokens_mistral': 909, 'n_tokens_neox': 892, 'n_words': 248}