<commit_before>import logging
import re
from collections import deque
from datetime import datetime

import requests
from praw.helpers import submission_stream
from praw.errors import AlreadySubmitted, APIException, HTTPException

from images_of import settings
from images_of.subreddit import Subreddit

RETRY_MINUTES = 3
LOG = logging.getLogger(__name__)

class Bot:
    def __init__(self, r, should_post=True):
        self.r = r
        self.should_post = should_post
        self.recent_posts = deque(maxlen=50)

        LOG.info('Loading global user blacklist from wiki')
        self.blacklist_users = self._read_blacklist('userblacklist')

        LOG.info('Loading global subreddit blacklist from wiki')
        self.blacklist_subs = self._read_blacklist('subredditblacklist')

        self.subreddits = []
        for sub_settings in settings.SLAVE_SUBS:
            sub = Subreddit(**sub_settings)
            sub.load_wiki_blacklist(r)
            self.subreddits.append(sub)

        ext_pattern = '({})$'.format('|'.join(settings.EXTENSIONS))
        self.ext_re = re.compile(ext_pattern, flags=re.IGNORECASE)

        domain_pattern = '^({})$'.format('|'.join(settings.DOMAINS))
        self.domain_re = re.compile(domain_pattern, flags=re.IGNORECASE)

    def _read_blacklist(self, wiki_page):
        content = self.r.get_wiki_page(settings.MASTER_SUB, wiki_page).content_md
        entries = [line.strip().lower()[3:] for line in content.splitlines() if line]
        return set(entries)

    def check(self, post):
        """Check global conditions on a post"""
        if not settings.NSFW_OK and post.over_18:
            return False

        user = post.author.name.lower()
        if user in self.blacklist_users:
            return False

        sub = post.subreddit.display_name.lower()
        if sub in self.blacklist_subs:
            return False

        if self.domain_re.search(post.domain):
            return True

        if self.ext_re.search(post.url):
            return True

        return False


    def crosspost(self, post, sub):
        title = post.title
        comment = '[Original post]({}) by /u/{} in /r/{}'.format(
                post.permalink,
                post.author,
                post.subreddit)

        log_entry = (post.url, sub.name)
        if log_entry in self.recent_posts:
            LOG.info('Already posted {} to /r/{}. Skipping.'.format(title, sub.name))
            return
        else:
            self.recent_posts.append(log_entry)
            LOG.debug('Added {} to recent posts. Now tracking {} items.'
                          .format(log_entry, len(self.recent_posts)))

        try:
            LOG.info('X-Posting into /r/{}: {}'.format(sub.name, title))
            if self.should_post:
                xpost = self.r.submit(
                            sub.name,
                            title,
                            url=post.url,
                            captcha=None,
                            send_replies=True,
                            resubmit=False)

            LOG.debug('Commenting: {}'.format(comment))
            if self.should_post:
                xpost.add_comment(comment)

        except AlreadySubmitted:
            LOG.info('Already submitted. Skipping.')
        except APIException as e:
            LOG.warning(e)
        

    def verify_age(self, post):
        if hasattr(post, 'age_verified'):
            return True

        created = datetime.utcfromtimestamp(post.author.created_utc)
        age = (datetime.utcnow() - created).days
        if age > 2:
            post.age_verified = True
            return True
        return False

    def _do_post(self, post):
        if not self.check(post):
            return

        for sub in self.subreddits:
            if sub.check(post):
                if not self.verify_age(post):
                    return
                self.crosspost(post, sub)

    def run(self):
        stream = submission_stream(self.r, 'all', verbosity=0)

        while True:
            try:
                for post in stream:
                    self._do_post(post)
            except HTTPException as e:
                LOG.warning('Reddit is down. Sleeping.')
                LOG.debug(e)
                sleep(60 * RETRY_MINUTES)
                continue
            except requests.ReadTimeout as e:
                LOG.warning('Timed out reading Reddit data. Sleeping.')
                sleep(60 * RETRY_MINUTES)
                continue
            except requests.ConnectionError as e:
                LOG.warning('Connection failed - trying again.')
                continue

            LOG.warning('Stream ended. Restarting.')
<commit_msg>Fix sleep, adjust repeat log.<commit_after>import logging
import re
from collections import deque
from datetime import datetime
from time import sleep

import requests
from praw.helpers import submission_stream
from praw.errors import AlreadySubmitted, APIException, HTTPException

from images_of import settings
from images_of.subreddit import Subreddit

RETRY_MINUTES = 2
LOG = logging.getLogger(__name__)

class Bot:
    def __init__(self, r, should_post=True):
        self.r = r
        self.should_post = should_post
        self.recent_posts = deque(maxlen=50)

        LOG.info('Loading global user blacklist from wiki')
        self.blacklist_users = self._read_blacklist('userblacklist')

        LOG.info('Loading global subreddit blacklist from wiki')
        self.blacklist_subs = self._read_blacklist('subredditblacklist')

        self.subreddits = []
        for sub_settings in settings.SLAVE_SUBS:
            sub = Subreddit(**sub_settings)
            sub.load_wiki_blacklist(r)
            self.subreddits.append(sub)

        ext_pattern = '({})$'.format('|'.join(settings.EXTENSIONS))
        self.ext_re = re.compile(ext_pattern, flags=re.IGNORECASE)

        domain_pattern = '^({})$'.format('|'.join(settings.DOMAINS))
        self.domain_re = re.compile(domain_pattern, flags=re.IGNORECASE)

    def _read_blacklist(self, wiki_page):
        content = self.r.get_wiki_page(settings.MASTER_SUB, wiki_page).content_md
        entries = [line.strip().lower()[3:] for line in content.splitlines() if line]
        return set(entries)

    def check(self, post):
        """Check global conditions on a post"""
        if not settings.NSFW_OK and post.over_18:
            return False

        user = post.author.name.lower()
        if user in self.blacklist_users:
            return False

        sub = post.subreddit.display_name.lower()
        if sub in self.blacklist_subs:
            return False

        if self.domain_re.search(post.domain):
            return True

        if self.ext_re.search(post.url):
            return True

        return False


    def crosspost(self, post, sub):
        title = post.title
        comment = '[Original post]({}) by /u/{} in /r/{}'.format(
                post.permalink,
                post.author,
                post.subreddit)

        log_entry = (post.url, sub.name)
        if log_entry in self.recent_posts:
            # put it back at the end of the queue
            self.recent_posts.remove(log_entry)
            self.recent_posts.append(log_entry)
            LOG.info('Already posted {} to /r/{}. Skipping.'.format(title, sub.name))
            return
        else:
            self.recent_posts.append(log_entry)
            LOG.debug('Added {} to recent posts. Now tracking {} items.'
                          .format(log_entry, len(self.recent_posts)))

        try:
            LOG.info('X-Posting into /r/{}: {}'.format(sub.name, title))
            if self.should_post:
                xpost = self.r.submit(
                            sub.name,
                            title,
                            url=post.url,
                            captcha=None,
                            send_replies=True,
                            resubmit=False)

            LOG.debug('Commenting: {}'.format(comment))
            if self.should_post:
                xpost.add_comment(comment)

        except AlreadySubmitted:
            LOG.info('Already submitted. Skipping.')
        except APIException as e:
            LOG.warning(e)
        

    def verify_age(self, post):
        if hasattr(post, 'age_verified'):
            return True

        created = datetime.utcfromtimestamp(post.author.created_utc)
        age = (datetime.utcnow() - created).days
        if age > 2:
            post.age_verified = True
            return True
        return False

    def _do_post(self, post):
        if not self.check(post):
            return

        for sub in self.subreddits:
            if sub.check(post):
                if not self.verify_age(post):
                    return
                self.crosspost(post, sub)

    def run(self):
        stream = submission_stream(self.r, 'all', verbosity=0)

        while True:
            try:
                for post in stream:
                    self._do_post(post)
            except HTTPException as e:
                LOG.warning('Reddit is down. Sleeping.')
                LOG.debug(e)
                sleep(60 * RETRY_MINUTES)
                continue
            except requests.ReadTimeout as e:
                LOG.warning('Timed out reading Reddit data. Sleeping.')
                LOG.debug(e)
                sleep(60 * RETRY_MINUTES)
                continue
            except requests.ConnectionError as e:
                LOG.warning('Connection failed. Restarting.')
                LOG.debug(e)
                continue

            LOG.warning('Stream ended. Restarting.')

{'fraction_non_alphanumeric': 0.09469227591904172, 'fraction_numerical': 0.0024783147459727386, 'mean_word_length': 1.7545506257110353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '181577', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2643, 'n_words': 630}