<commit_before>from __future__ import division

import logging
import numpy as np
import os
import random
import signal
import sys
import time

from multiprocessing import Pool
from tweepy import Cursor

from . import api, config
from .poetry_percentile_rank import PercentilePoetryRanker


logger = logging.getLogger(__name__)
signal.signal(signal.SIGINT, lambda signal, frame: sys.exit(0))


ranker = PercentilePoetryRanker(
    config.get("Ranker", "inquirer"),
    config.get("Ranker", "coca"),
    config.get("Ranker", "percentiles"))


def rank(tweet):
    """ Rank each tweet.

    This function must be pickleable. This acheived by making this a
    module level function.
    """

    pid = os.getpid()
    np.random.seed(pid)
    random.seed(pid)
    return ranker.rank([tweet.text])


class TweetPoller(object):

    pool = Pool(processes=int(config.get("Poller", "processes")))

    poll_interval = int(config.get("Poller", "interval"))

    q = config.get("Poller", "query")

    count = config.get("Poller", "count")

    since_id = None

    def sleep_until(self, nextpoll):
        sleeptime = nextpoll - time.time()
        if sleeptime > 0:
            logger.info("Done. Waiting for %s seconds.", sleeptime)
            time.sleep(sleeptime)<commit_msg>Do not repeat tweets on restart.<commit_after>from __future__ import division

import ConfigParser
import logging
import numpy as np
import os
import random
import signal
import sys
import time

from multiprocessing import Pool
from tweepy import Cursor

from . import api, config, config_path
from .poetry_percentile_rank import PercentilePoetryRanker


logger = logging.getLogger(__name__)
signal.signal(signal.SIGINT, lambda signal, frame: sys.exit(0))


ranker = PercentilePoetryRanker(
    config.get("Ranker", "inquirer"),
    config.get("Ranker", "coca"),
    config.get("Ranker", "percentiles"))


def rank(tweet):
    """ Rank each tweet.

    This function must be pickleable. This acheived by making this a
    module level function.
    """

    pid = os.getpid()
    np.random.seed(pid)
    random.seed(pid)
    return ranker.rank([tweet.text])


class TweetPoller(object):

    pool = Pool(processes=int(config.get("Poller", "processes")))

    poll_interval = int(config.get("Poller", "interval"))

    q = config.get("Poller", "query")

    count = config.get("Poller", "count")

    @property
    def since_id(self):
        try:
            return config.get("Twitter", "since_id")
        except ConfigParser.NoOptionError:
            return None

    @since_id.setter
    def since_id(self, id):
        config.set("Twitter", "since_id", id)
        with open(config_path, "wb") as configfile:
            config.write(configfile)

    def sleep_until(self, nextpoll):
        sleeptime = nextpoll - time.time()
        if sleeptime > 0:
            logger.info("Done. Waiting for %s seconds.", sleeptime)
            time.sleep(sleeptime)
{'fraction_non_alphanumeric': 0.1145975443383356, 'fraction_numerical': 0.001364256480218281, 'mean_word_length': 3.384155455904335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '64291', 'n_tokens_mistral': 956, 'n_tokens_neox': 902, 'n_words': 245}