<commit_before>#!/usr/bin/env python3


import argparse
import csv
import datetime
import os
import re
import time

import malaffinity
import praw
import requests


wait_between_requests = 2
retry_after_failed_request = 5

verbose = False
search_comment_body = False


regex = "myanimelist\.net/(?:profile|animelist)/([a-z0-9_-]+)"
regex = re.compile(regex, re.I)


# Set the pearson stuff up.
# Too lazy to rename everything and update docs. Sorry
pearson = malaffinity.MALAffinity(round=2)


def vprint(message, end="\n"):
    if verbose:
        print(message, end=end)
    return


def _retrieve_comment_ids(submission_id):
    # https://www.reddit.com/r/redditdev/comments/5to97v/slug/ddoeedj/
    comments = requests.request(
        "GET",
        "http://apiv2.pushshift.io/reddit/get/comment_ids/{}"
            .format(submission_id)
    )
    comments = comments.json()

    return comments["data"]

def _create_reddit_instance():
    return praw.Reddit("reddit")


def get_comment_stream():
    reddit = _create_reddit_instance()

    return reddit.subreddit("anime").stream.comments()

def get_comments_from_submission(submission_id):
    reddit = _create_reddit_instance()

    comments = _retrieve_comment_ids(submission_id)

    return reddit.info(comments)

def get_comments_from_ftfs():
    reddit = _create_reddit_instance()
    subreddit = reddit.subreddit("anime")

    # Manually look up for FTFs because it seems the search function misses
    # some of them out sometimes. Dunno why, probably something on their side.
    # The search function is rubbish anyway so it's hardly surprising...
    # TODO: better var names
    d = datetime.date.today()
    e = datetime.date(d.year, 1, 1)

    comments = []

    # TODO: check this works properly
    # TODO: There has to be a better way of doing this...
    # TODO: Clean this up
    while d > e:
        # Friday check. Friday = index 4
        # TODO: Make this easier to understand
        if d.weekday() % 7 == 4:
            # Tis a friday. Form the title
            # "Free Talk Fridays - Week of MONTH DATE, YEAR"
            ftf_title = d.strftime("Free Talk Fridays - Week of %B %d, %Y")

            # Try not to make Reddit and Pushshift's servers angry
            time.sleep(2)

            done = False

            # WHY CAN'T YOU JUST WORK, REDDIT SEARCH, YOU PILE OF SHITE
            while not done:
                try:
                    ftf = subreddit.search(ftf_title)

                    ftf = list(ftf)[0]

                    # Add comment ids to the comments list
                    comments += _retrieve_comment_ids(ftf.id)
                except:
                    print(
                        "ERR: Failed to retrieve comment ids for FTF: {}. Retrying in 30 seconds."
                            .format(ftf_title)
                    )
                    time.sleep(30)
                else:
                    done = True

        # Decrease the day
        d -= datetime.timedelta(days=1)

    # Sorry, reddit servers
    return reddit.info(comments)


def handle_comment(comment):<commit_msg>More verbose when finding comment ids for ftf option<commit_after>#!/usr/bin/env python3


import argparse
import csv
import datetime
import os
import re
import time

import malaffinity
import praw
import requests


wait_between_requests = 2
retry_after_failed_request = 5

verbose = False
search_comment_body = False


regex = "myanimelist\.net/(?:profile|animelist)/([a-z0-9_-]+)"
regex = re.compile(regex, re.I)


# Set the pearson stuff up.
# Too lazy to rename everything and update docs. Sorry
pearson = malaffinity.MALAffinity(round=2)


def vprint(message, end="\n"):
    if verbose:
        print(message, end=end)
    return


def _retrieve_comment_ids(submission_id):
    # https://www.reddit.com/r/redditdev/comments/5to97v/slug/ddoeedj/
    comments = requests.request(
        "GET",
        "http://apiv2.pushshift.io/reddit/get/comment_ids/{}"
            .format(submission_id)
    )
    comments = comments.json()

    return comments["data"]

def _create_reddit_instance():
    return praw.Reddit("reddit")


def get_comment_stream():
    reddit = _create_reddit_instance()

    return reddit.subreddit("anime").stream.comments()

def get_comments_from_submission(submission_id):
    reddit = _create_reddit_instance()

    comments = _retrieve_comment_ids(submission_id)

    return reddit.info(comments)

def get_comments_from_ftfs():
    reddit = _create_reddit_instance()
    subreddit = reddit.subreddit("anime")

    # Manually look up for FTFs because it seems the search function misses
    # some of them out sometimes. Dunno why, probably something on their side.
    # The search function is rubbish anyway so it's hardly surprising...
    # TODO: better var names
    d = datetime.date.today()
    e = datetime.date(d.year, 1, 1)

    comments = []

    # TODO: check this works properly
    # TODO: There has to be a better way of doing this...
    # TODO: Clean this up
    while d > e:
        # Friday check. Friday = index 4
        # TODO: Make this easier to understand
        if d.weekday() % 7 == 4:
            # Tis a friday. Form the title
            # "Free Talk Fridays - Week of MONTH DATE, YEAR"
            ftf_title = d.strftime("Free Talk Fridays - Week of %B %d, %Y")

            # Try not to make Reddit and Pushshift's servers angry
            time.sleep(2)

            done = False

            # WHY CAN'T YOU JUST WORK, REDDIT SEARCH, YOU PILE OF SHITE
            while not done:
                try:
                    print("Retrieving comment ids for FTF: {}...".format(ftf_title))

                    ftf = subreddit.search(ftf_title)

                    ftf = list(ftf)[0]

                    # Add comment ids to the comments list
                    comments += _retrieve_comment_ids(ftf.id)
                except:
                    print(
                        "ERR: Failed to retrieve comment ids for FTF: {}. Retrying in 30 seconds."
                            .format(ftf_title)
                    )
                    time.sleep(30)
                else:
                    done = True

        # Decrease the day
        d -= datetime.timedelta(days=1)

    # Sorry, reddit servers
    return reddit.info(comments)


def handle_comment(comment):
{'fraction_non_alphanumeric': 0.09131051884560795, 'fraction_numerical': 0.006938968616937391, 'mean_word_length': 2.193353474320242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4848940', 'n_tokens_mistral': 1956, 'n_tokens_neox': 1854, 'n_words': 553}