<commit_before>"""Utility to provide submission and comment statistics in a subreddit."""

from __future__ import print_function
import codecs
import re
import sys
import time
import webbrowser
from collections import defaultdict
from datetime import datetime
from praw import Reddit
from praw.errors import ExceptionList, RateLimitExceeded
from praw.helpers import flatten_tree
from praw.objects import Redditor
from requests.exceptions import HTTPError
from six import iteritems, itervalues, text_type as tt
from update_checker import update_check
from . import __version__
from .helpers import arg_parser

DAYS_IN_SECONDS = 60 * 60 * 24
MAX_BODY_SIZE = 40000


def safe_title(submission):
    """Return titles with newlines replaced by spaces and stripped."""
    return submission.title.replace('\n', ' ').strip()


class SubRedditStats(object):

    """Contain all the functionality of the subreddit_stats command."""

    post_prefix = tt('Subreddit Stats:')
    post_header = tt('---\n###{0}\n')
    post_footer = tt('>Generated with [BBoe](/u/bboe)\'s [Subreddit Stats]'
                     '(https://github.com/praw-dev/prawtools)  \n{0}'
                     'SRS Marker: {1}')
    re_marker = re.compile(r'SRS Marker: (\d+)')

    @staticmethod
    def _previous_max(submission):
        try:
            val = SubRedditStats.re_marker.findall(submission.selftext)[-1]
            return float(val)
        except (IndexError, TypeError):
            print('End marker not found in previous submission. Aborting')
            sys.exit(1)

    @staticmethod
    def _permalink(permalink):
        tokens = permalink.split('/')
        if tokens[8] == '':  # submission
            return tt('/{0}').format(tokens[6])
        else:  # comment
            return tt('/comments/{0}/_/{1}?context=999').format(tokens[6],
                                                              tokens[8])

    @staticmethod
    def _pts(points):
        return '1 pt' if points == 1 else '{0} pts'.format(points)

    @staticmethod
    def _user(user):
        if user is None:
            return '_deleted_'
        elif isinstance(user, Redditor):
            user = str(user)
        return tt('u/{1}').format(user.replace('_', r'\_'), user)

    @staticmethod
    def _submit(func, *args, **kwargs):
        def sleep(sleep_time):
            print('\tSleeping for {0} seconds'.format(sleep_time))
            time.sleep(sleep_time)

        while True:
            try:
                return func(*args, **kwargs)
            except RateLimitExceeded as error:
                sleep(error.sleep_time)
            except ExceptionList as exception_list:
                for error in exception_list.errors:
                    if isinstance(error, RateLimitExceeded):
                        sleep(error.sleep_time)
                        break
                else:
                    raise

    def __init__(self, subreddit, site, verbosity, distinguished):
        """Initialize the SubRedditStats instance with config options."""
        self.reddit = Reddit(str(self), site, disable_update_check=True)
        self.subreddit = self.reddit.get_subreddit(subreddit)
        self.verbosity = verbosity
        self.distinguished = distinguished
        self.submissions = []
        self.comments = []
        self.submitters = defaultdict(list)
        self.commenters = defaultdict(list)
        self.min_date = 0
        self.max_date = time.time() - DAYS_IN_SECONDS * 3<commit_msg>, api_request_delay=1<commit_after>"""Utility to provide submission and comment statistics in a subreddit."""

from __future__ import print_function
import codecs
import re
import sys
import time
import webbrowser
from collections import defaultdict
from datetime import datetime
from praw import Reddit
from praw.errors import ExceptionList, RateLimitExceeded
from praw.helpers import flatten_tree
from praw.objects import Redditor
from requests.exceptions import HTTPError
from six import iteritems, itervalues, text_type as tt
from update_checker import update_check
from . import __version__
from .helpers import arg_parser

DAYS_IN_SECONDS = 60 * 60 * 24
MAX_BODY_SIZE = 40000


def safe_title(submission):
    """Return titles with newlines replaced by spaces and stripped."""
    return submission.title.replace('\n', ' ').strip()


class SubRedditStats(object):

    """Contain all the functionality of the subreddit_stats command."""

    post_prefix = tt('Subreddit Stats:')
    post_header = tt('---\n###{0}\n')
    post_footer = tt('>Generated with [BBoe](/u/bboe)\'s [Subreddit Stats]'
                     '(https://github.com/praw-dev/prawtools)  \n{0}'
                     'SRS Marker: {1}')
    re_marker = re.compile(r'SRS Marker: (\d+)')

    @staticmethod
    def _previous_max(submission):
        try:
            val = SubRedditStats.re_marker.findall(submission.selftext)[-1]
            return float(val)
        except (IndexError, TypeError):
            print('End marker not found in previous submission. Aborting')
            sys.exit(1)

    @staticmethod
    def _permalink(permalink):
        tokens = permalink.split('/')
        if tokens[8] == '':  # submission
            return tt('/{0}').format(tokens[6])
        else:  # comment
            return tt('/comments/{0}/_/{1}?context=999').format(tokens[6],
                                                              tokens[8])

    @staticmethod
    def _pts(points):
        return '1 pt' if points == 1 else '{0} pts'.format(points)

    @staticmethod
    def _user(user):
        if user is None:
            return '_deleted_'
        elif isinstance(user, Redditor):
            user = str(user)
        return tt('u/{1}').format(user.replace('_', r'\_'), user)

    @staticmethod
    def _submit(func, *args, **kwargs):
        def sleep(sleep_time):
            print('\tSleeping for {0} seconds'.format(sleep_time))
            time.sleep(sleep_time)

        while True:
            try:
                return func(*args, **kwargs)
            except RateLimitExceeded as error:
                sleep(error.sleep_time)
            except ExceptionList as exception_list:
                for error in exception_list.errors:
                    if isinstance(error, RateLimitExceeded):
                        sleep(error.sleep_time)
                        break
                else:
                    raise

    def __init__(self, subreddit, site, verbosity, distinguished):
        """Initialize the SubRedditStats instance with config options."""
        self.reddit = Reddit(str(self), site, disable_update_check=True, api_request_delay=1)
        self.subreddit = self.reddit.get_subreddit(subreddit)
        self.verbosity = verbosity
        self.distinguished = distinguished
        self.submissions = []
        self.comments = []
        self.submitters = defaultdict(list)
        self.commenters = defaultdict(list)
        self.min_date = 0
        self.max_date = time.time() - DAYS_IN_SECONDS * 3
{'fraction_non_alphanumeric': 0.10361067503924647, 'fraction_numerical': 0.009704581133152562, 'mean_word_length': 2.3643783005280845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2803477', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1952, 'n_words': 521}