<commit_before>#!/usr/bin/env python

"""
Based on what I learned creeping the source code for
https://github.com/cris9696/PlayStoreLinks_Bot
"""
import praw         # reddit wrapper
import time         # obvious
import pickle       # dump list and dict to file
import os           # OS-related stuff
import sys          # ""
import atexit       # To handle unexpected crashes or just normal exiting
import logging      # logging
import signal       # Catch SIGINT
import string       # Used in generating random strings
import random       # ""
import requests     # For URL requests, ued in gfycat API
import urllib       # For encoding urls
import subprocess   # To send shell commands, used for local testing

# tagline
commentTag = "------\n\n*Hi! I'm a bot created to x-post gifs/vines/gfycats" + \
             " from /r/coys over to /r/SpursGifs.*\n\n*Feedback/bug reports? Send a" + \
             " message to [pandanomic](http://www.reddit.com/message/compose?to=pan" + \
             "danomic).*\n\n*[Source code](https://github.com/pandanomic/Spurs" + \
             "Gifs_xposterbot)*"

# DB for caching previous posts
dbFile = "spursgifs_xposterDB"

# File with login credentials
propsFile = "login.properties"

# flag to check if db file's already checked
fileOpened = False

# subreddit to x-post to. Changes if testing
postSub = "SpursGifs"

# for cron jobs
cron = False

# for my mac, I use terminal-notifier to get updates
macUpdate = False

# for keeping track of if we're on Heroku
running_on_heroku = False

if os.environ.get('MEMCACHEDCLOUD_SERVERS', None):
    # import bmemcached
    import pylibmc

    print '\tRunning on heroku, using memcached'

    running_on_heroku = True
    # mc = bmemcached.Client(os.environ.get('MEMCACHEDCLOUD_SERVERS').split(','),
    #                        os.environ.get('MEMCACHEDCLOUD_USERNAME'), os.environ.get('MEMCACHEDCLOUD_PASSWORD'))
    mc = pylibmc.Client(os.environ.get('MEMCACHEDCLOUD_SERVERS'),
                        username=os.environ.get('MEMCACHEDCLOUD_USERNAME'),
                        password=os.environ.get('MEMCACHEDCLOUD_PASSWORD'))


# Called when exiting the program
def exit_handler():
    print "(SHUTTING DOWN)"
    if not running_on_heroku and fileOpened:
        pickle.dump(already_done, f)
        f.close()
    os.remove("BotRunning")


# Called on SIGINT
def signal_handler(signal, frame):
    print '\n(Caught SIGINT, exiting gracefully...)'
    sys.exit()

# Register the function that get called on exit
atexit.register(exit_handler)

# Register function to call on SIGINT
signal.signal(signal.SIGINT, signal_handler)


# Function to exit the bot
def exit_bot():
    sys.exit()


# Main bot runner
def bot():
    print "(Parsing new 30)"
    new_count = 0
    for submission in coys_subreddit.get_new(limit=30):
        if validate_submission(submission):
            if running_on_heroku:
                # mc.set(str(submission.id), "True")
                # assert mc.get(str(submission.id)) == "True"
                mc[str(submission.id)] = "True"
                assert (str(submission.id)) in mc
            else:
                already_done.append(submission.id)
            print "(New Post)"
            submit(spursgifs_subreddit, submission)

    if new_count == 0:
        print "(Nothing new)"


# Submission
def submit(subreddit, submission):
    if macUpdate:
        print '\tNotifying on Mac'
        try:
            subprocess.call(
                ["terminal-notifier", "-message", "New post", "-title", "Spurs Gif Bot", "-sound", "default"])
        except OSError:
            print '\t--Could not find terminal-notifier, please reinstall'

    url_to_submit = submission.url

    gfy_converted = False

    # Convert if it's a gif
    if extension(submission.url) == ".gif":
        new_url_to_submit = gfycat_convert(url_to_submit)
        if new_url_to_submit != "Error":
            url_to_submit = new_url_to_submit
            gfy_converted = True

    print "\tSubmitting to /r/" + postSub + "..."
    try:
        new_submission = subreddit.submit(
            submission.title + " (x-post from /r/coys)", url=url_to_submit)

        if gfy_converted:
            if running_on_heroku:
                # mc.set(str(new_submission.id), "True")
                # assert mc.get(str(new_submission.id)) == "True"
                mc[str(new_submission.id)] = "True"
                assert (str(new_submission.id)) in mc
            else:
                already_done.append(new_submission.id)

        followup_comment(submission, new_submission, gfy_converted)
    except praw.errors.AlreadySubmitted:
        # logging.exception("Already submitted")
        print "\t--Already submitted, caching"
        if running_on_heroku:
            # mc.set(str(submission.id), "True")
            # assert mc.get(str(submission.id)) == "True"
            mc[str(submission.id)] = "True"
            assert (str(submission.id)) in mc
        else:
            if submission.id not in already_done:
                already_done.append(submission.id)
    except praw.errors.RateLimitExceeded:
        print "\t--Rate Limit Exceeded"
        if running_on_heroku:
            # mc.delete(str(submission.id))
            del mc[str(submission.id)]
        else:
            already_done.remove(submission.id)
    except praw.errors.APIException:
        logging.exception("Error on link submission.")


# Retrieves the extension
def extension(url):
    return os.path.splitext(url)[1]


# Validates if a submission should be posted
def validate_submission(submission):
    # check domain and extension validity
    if submission.domain in allowedDomains or extension(submission.url) in allowedExtensions:
        # Running on heroku, check the memcache
        if running_on_heroku:
            # obj = mc.get(str(submission.id))
            # if not obj:
            #     return True
            if str(submission.id) not in mc:
                return True
        if submission.id not in already_done:
            return True
    return False


# Followup Comment
def followup_comment(submission, new_submission, gfy_converted):
    print("\tFollowup Comment...")
    # user = r.get_redditor("spursgifs_xposterbot")
    # new_submission = user.get_submitted(limit=1).next()
    followup_comment_text = "Originally posted [here](" + \
                            submission.permalink + ") by /u/" + \
                            submission.author.name + \
                            ".\n\n"
    followup_comment_text += commentTag

    try:
        new_submission.add_comment(followup_comment_text)
        notify_comment(new_submission.permalink, submission, gfy_converted)
    except praw.errors.RateLimitExceeded:
        print "\t--Rate Limit Exceeded"
    except praw.errors.APIException:
        logging.exception("Error on followupComment")
<commit_msg>Pylibmc was a bust because it wants django, back to bmemcached<commit_after>#!/usr/bin/env python

"""
Based on what I learned creeping the source code for
https://github.com/cris9696/PlayStoreLinks_Bot
"""
import praw         # reddit wrapper
import time         # obvious
import pickle       # dump list and dict to file
import os           # OS-related stuff
import sys          # ""
import atexit       # To handle unexpected crashes or just normal exiting
import logging      # logging
import signal       # Catch SIGINT
import string       # Used in generating random strings
import random       # ""
import requests     # For URL requests, ued in gfycat API
import urllib       # For encoding urls
import subprocess   # To send shell commands, used for local testing

# tagline
commentTag = "------\n\n*Hi! I'm a bot created to x-post gifs/vines/gfycats" + \
             " from /r/coys over to /r/SpursGifs.*\n\n*Feedback/bug reports? Send a" + \
             " message to [pandanomic](http://www.reddit.com/message/compose?to=pan" + \
             "danomic).*\n\n*[Source code](https://github.com/pandanomic/Spurs" + \
             "Gifs_xposterbot)*"

# DB for caching previous posts
dbFile = "spursgifs_xposterDB"

# File with login credentials
propsFile = "login.properties"

# flag to check if db file's already checked
fileOpened = False

# subreddit to x-post to. Changes if testing
postSub = "SpursGifs"

# for cron jobs
cron = False

# for my mac, I use terminal-notifier to get updates
macUpdate = False

# for keeping track of if we're on Heroku
running_on_heroku = False

if os.environ.get('MEMCACHEDCLOUD_SERVERS', None):
    import bmemcached
    # import pylibmc

    print '\tRunning on heroku, using memcached'

    running_on_heroku = True
    mc = bmemcached.Client(os.environ.get('MEMCACHEDCLOUD_SERVERS').split(','),
                           os.environ.get('MEMCACHEDCLOUD_USERNAME'), os.environ.get('MEMCACHEDCLOUD_PASSWORD'))
    # mc = pylibmc.Client(os.environ.get('MEMCACHEDCLOUD_SERVERS'),
    #                     username=os.environ.get('MEMCACHEDCLOUD_USERNAME'),
    #                     password=os.environ.get('<PASSWORD>'))


# Called when exiting the program
def exit_handler():
    print "(SHUTTING DOWN)"
    if not running_on_heroku and fileOpened:
        pickle.dump(already_done, f)
        f.close()
    os.remove("BotRunning")


# Called on SIGINT
def signal_handler(signal, frame):
    print '\n(Caught SIGINT, exiting gracefully...)'
    sys.exit()

# Register the function that get called on exit
atexit.register(exit_handler)

# Register function to call on SIGINT
signal.signal(signal.SIGINT, signal_handler)


# Function to exit the bot
def exit_bot():
    sys.exit()


# Main bot runner
def bot():
    print "(Parsing new 30)"
    new_count = 0
    for submission in coys_subreddit.get_new(limit=30):
        if validate_submission(submission):
            if running_on_heroku:
                mc.set(str(submission.id), "True")
                assert str(mc.get(str(submission.id))) == "True"
                print '\tCached ' + str(submission.id) + ': ' + str(mc.get(str(submission.id)))
                # mc[str(submission.id)] = "True"
                # assert (str(submission.id)) in mc
            else:
                already_done.append(submission.id)
            print "(New Post)"
            submit(spursgifs_subreddit, submission)

    if new_count == 0:
        print "(Nothing new)"


# Submission
def submit(subreddit, submission):
    if macUpdate:
        print '\tNotifying on Mac'
        try:
            subprocess.call(
                ["terminal-notifier", "-message", "New post", "-title", "Spurs Gif Bot", "-sound", "default"])
        except OSError:
            print '\t--Could not find terminal-notifier, please reinstall'

    url_to_submit = submission.url

    gfy_converted = False

    # Convert if it's a gif
    if extension(submission.url) == ".gif":
        new_url_to_submit = gfycat_convert(url_to_submit)
        if new_url_to_submit != "Error":
            url_to_submit = new_url_to_submit
            gfy_converted = True

    print "\tSubmitting to /r/" + postSub + "..."
    try:
        new_submission = subreddit.submit(
            submission.title + " (x-post from /r/coys)", url=url_to_submit)

        if gfy_converted:
            if running_on_heroku:
                mc.set(str(new_submission.id), "True")
                assert str(mc.get(str(new_submission.id))) == "True"
                print '\tCached ' + str(new_submission.id) + ': ' + str(mc.get(str(new_submission.id)))
                # mc[str(new_submission.id)] = "True"
                # assert (str(new_submission.id)) in mc
            else:
                already_done.append(new_submission.id)

        followup_comment(submission, new_submission, gfy_converted)
    except praw.errors.AlreadySubmitted:
        # logging.exception("Already submitted")
        print "\t--Already submitted, caching"
        if running_on_heroku:
            mc.set(str(submission.id), "True")
            assert str(mc.get(str(submission.id))) == "True"
            print '\tCached ' + str(submission.id) + ': ' + str(mc.get(str(submission.id)))
            # mc[str(submission.id)] = "True"
            # assert (str(submission.id)) in mc
        else:
            if submission.id not in already_done:
                already_done.append(submission.id)
    except praw.errors.RateLimitExceeded:
        print "\t--Rate Limit Exceeded"
        if running_on_heroku:
            mc.delete(str(submission.id))
            print '\tDeleted ' + str(submission.id)
            # del mc[str(submission.id)]
        else:
            already_done.remove(submission.id)
    except praw.errors.APIException:
        logging.exception("Error on link submission.")


# Retrieves the extension
def extension(url):
    return os.path.splitext(url)[1]


# Validates if a submission should be posted
def validate_submission(submission):
    # check domain and extension validity
    if submission.domain in allowedDomains or extension(submission.url) in allowedExtensions:
        # Running on heroku, check the memcache
        if running_on_heroku:
            print '\tChecking cache for ' + str(submission.id)
            obj = mc.get(str(submission.id))
            if not obj:
                print '\t--id not present'
                return True
            # if str(submission.id) not in mc:
            #     return True
        if submission.id not in already_done:
            return True
    return False


# Followup Comment
def followup_comment(submission, new_submission, gfy_converted):
    print("\tFollowup Comment...")
    # user = r.get_redditor("spursgifs_xposterbot")
    # new_submission = user.get_submitted(limit=1).next()
    followup_comment_text = "Originally posted [here](" + \
                            submission.permalink + ") by /u/" + \
                            submission.author.name + \
                            ".\n\n"
    followup_comment_text += commentTag

    try:
        new_submission.add_comment(followup_comment_text)
        notify_comment(new_submission.permalink, submission, gfy_converted)
    except praw.errors.RateLimitExceeded:
        print "\t--Rate Limit Exceeded"
    except praw.errors.APIException:
        logging.exception("Error on followupComment")

{'fraction_non_alphanumeric': 0.10407461953853707, 'fraction_numerical': 0.001683147485798443, 'mean_word_length': 2.4065934065934065, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1027659', 'n_tokens_mistral': 4429, 'n_tokens_neox': 4308, 'n_words': 1072}