<commit_before>#!/usr/bin/env python

import feedparser
import time
import pickle

import publishx
import config

import logging
import imp

from socket import error as SocketError
from termcolor import colored, cprint


def setup_logging(level):
    log = logging.getLogger('atomtopubsub')
    log.setLevel(level)
    ch = logging.StreamHandler()
    ch.setLevel(level)
    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    log.addHandler(ch)

setup_logging(logging.INFO)

parsed = {}
connected = False
xmpp = publishx.publishx(config)

# We feed the pubsub nodes
def parse():
    imp.reload(config)

    # We parse all the feeds
    for key, feed in config.feeds.items():
        print(colored('>> parsing %s' % key , 'magenta'))
        f = feedparser.parse(feed['url'])

        if(f.bozo == 1):
            print('XML Error')
            if(hasattr(f.bozo_exception, 'getMessage')):
                print(f.bozo_exception.getMessage())
            if(hasattr(f.bozo_exception, 'getLineNumber')):
                print('at line %s' % f.bozo_exception.getLineNumber())

        if(not key in parsed):
            xmpp.create(feed['server'], key, f.feed)

        # We check if we have some new entries
        for entry in f.entries:
            if key not in parsed or parsed[key] < entry.updated_parsed:
                print(colored('++ new entry %s' % entry.title, 'green'))
                time.sleep(2)
                xmpp.publish(feed['server'], key, entry)
            else:
                print(colored('++ update entry %s' % entry.title, 'yellow'))

        # And we update the last updated date for the feed
        if(f is not None and hasattr(f, 'updated_parsed')) :
            parsed[key] = f.updated_parsed
        else:
            print(colored('-- Parse failed for %s' % key, 'red'))

        save()

        # We distribute the parsing
        print(colored('Parsing next feed in %.2f minutes' % (float(config.refresh_time)/len(config.feeds)), 'cyan'))
        time.sleep((float(config.refresh_time) * 60)/len(config.feeds))

def load():
    try:
        pkl_file = open('cache.pkl', 'rb')
        parsed = pickle.load(pkl_file)
        pkl_file.close()
        return parsed
    except IOError:
        print('Creating the cache')
        return save()

def save():
    output = open('cache.pkl', 'wb')
    pickle.dump(parsed, output)
    output.close()
    return {}

parsed = load()
connected = xmpp.connect()
xmpp.process()

if(connected) :
    while(1):
        try:
            parse()

        except KeyboardInterrupt:
            xmpp.disconnect(wait=True)
            print("Exiting...")
            break<commit_msg>Remove global state, put it in function parameters.<commit_after>#!/usr/bin/env python

import feedparser
import time
import pickle

import publishx
import config

import logging
import imp

from socket import error as SocketError
from termcolor import colored, cprint


def setup_logging(level):
    log = logging.getLogger('atomtopubsub')
    log.setLevel(level)
    ch = logging.StreamHandler()
    ch.setLevel(level)
    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    log.addHandler(ch)

# We feed the pubsub nodes
def parse(parsed, xmpp):
    imp.reload(config)

    # We parse all the feeds
    for key, feed in config.feeds.items():
        print(colored('>> parsing %s' % key , 'magenta'))
        f = feedparser.parse(feed['url'])

        if(f.bozo == 1):
            print('XML Error')
            if(hasattr(f.bozo_exception, 'getMessage')):
                print(f.bozo_exception.getMessage())
            if(hasattr(f.bozo_exception, 'getLineNumber')):
                print('at line %s' % f.bozo_exception.getLineNumber())

        if(not key in parsed):
            xmpp.create(feed['server'], key, f.feed)

        # We check if we have some new entries
        for entry in f.entries:
            if key not in parsed or parsed[key] < entry.updated_parsed:
                print(colored('++ new entry %s' % entry.title, 'green'))
                time.sleep(2)
                xmpp.publish(feed['server'], key, entry)
            else:
                print(colored('++ update entry %s' % entry.title, 'yellow'))

        # And we update the last updated date for the feed
        if(f is not None and hasattr(f, 'updated_parsed')) :
            parsed[key] = f.updated_parsed
        else:
            print(colored('-- Parse failed for %s' % key, 'red'))

        save(parsed)

        # We distribute the parsing
        print(colored('Parsing next feed in %.2f minutes' % (float(config.refresh_time)/len(config.feeds)), 'cyan'))
        time.sleep((float(config.refresh_time) * 60)/len(config.feeds))

def load():
    try:
        pkl_file = open('cache.pkl', 'rb')
        parsed = pickle.load(pkl_file)
        pkl_file.close()
        return parsed
    except IOError:
        print('Creating the cache')
        return save({})

def save(parsed):
    output = open('cache.pkl', 'wb')
    pickle.dump(parsed, output)
    output.close()
    return {}

def main():
    setup_logging(logging.INFO)

    parsed = load()

    xmpp = publishx.publishx(config)
    connected = xmpp.connect()
    xmpp.process()

    if(connected) :
        while(1):
            try:
                parse(parsed, xmpp)

            except KeyboardInterrupt:
                xmpp.disconnect(wait=True)
                print("Exiting...")
                break

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.12479593687647379, 'fraction_numerical': 0.0021766733176129148, 'mean_word_length': 2.335753176043557, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1148697', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1665, 'n_words': 431}