<commit_before>#!/usr/bin/python2.7
# coding: utf-8


from __future__ import print_function
from localsettings import USERNAME, PASSWORD
import sqlite3
from pprint import pprint
import praw
import time
from Extrapolate import Extrapolate


class Storytellingbot(object):
    """
    Storytellingbot
    COGS-4640/CSCI-4977 Intelligent Virtual Agent
    https://github.com/theopak/storytellingbot
    https://github.com/reddit/reddit/wiki/API#rules
    """

    e = Extrapolate.Extrapolate()

    def __init__(self, username, password, db_file='local.db',
                 user_agent='Python:storytellingbot:v0.0.0 (by /u/theopakalyse)'):
        """
        Init data layer.
        Login to Reddit. OAuth2 bots gave a rate limit twice as high.
        """
        if DEBUG:
            print('[INFO] Storytellingbot.__init__() called…')
        self.con = sqlite3.connect(db_file)
        self.cur = self.con.cursor()
        self.setup()
        self.reddit = praw.Reddit(user_agent)
        self.reddit.login(username, password)
        if DEBUG:
            print('[INFO] Logged in as', username)

    def __del__(self):
        """
        Remove any locks on the data layer.
        """
        if DEBUG:
            print('[INFO] Storytellingbot.__del__() called…')
        self.con.close()
        if DEBUG:
            print('\tClosed db connection.')

    def setup(self):
        """
        Initialize bot implementation. This only needs to happen the first time
        but the function should be safe to call at any point.
        TODO(@theopak): Create a test subreddit if it does not exist.
        """
        query = '''
            CREATE TABLE IF NOT EXISTS outbox(
                number      integer PRIMARY KEY,-- our unique record identifier
                sent        datetime,           -- 0 if not sent, otherwise timestamp
                keyword     text,               -- keyword that was matched
                id          text,               -- reddit id (hash) of matched comment
                parentId    text,               -- reddit id (hash) of parent comment
                linkUrl     text,               -- permalink of the Submission
                body        text,               -- plaintext body of matched comment
                response    text                -- plaintext body to post as reply
            );'''
        self.cur.execute(query)
        query = '''
            CREATE TABLE IF NOT EXISTS keywords(
                word        text UNIQUE         -- one word or phrase to seach for
            );'''
        self.cur.execute(query)
        query = '''
            CREATE TABLE IF NOT EXISTS stories(
                id          integer PRIMARY KEY,-- our unique record identifier
                title       text,               -- plaintext story title
                content     text,               -- plaintext story
                source      text                -- URL
            );'''
        self.cur.execute(query)
        self.con.commit()
        if DEBUG:
            print('[INFO] Setup db')

        # Populate db
        self.cur.execute("SELECT Count() FROM keywords")
        number_of_rows = self.cur.fetchone()[0]
        if number_of_rows == 0:
            sample_keywords = ['unique_keyword_20150416', 'virtualagent']
            self.add_keywords(sample_keywords)
            if DEBUG:
                print('[INFO] Populated db')

    def add_keywords(self, keywords):
        if DEBUG:
            print('[INFO] Storytellingbot.add_keywords(): adding keywords to db:')
        for word in keywords:
            if DEBUG:
                print('\t' + word)
            self.cur.execute('INSERT into keywords VALUES (?)', (word,))
            self.con.commit()<commit_msg>minor: fixes typo in comment<commit_after>#!/usr/bin/python2.7
# coding: utf-8


from __future__ import print_function
from localsettings import USERNAME, PASSWORD
import sqlite3
from pprint import pprint
import praw
import time
from Extrapolate import Extrapolate


class Storytellingbot(object):
    """
    Storytellingbot
    COGS-4640/CSCI-4977 Intelligent Virtual Agent
    https://github.com/theopak/storytellingbot
    https://github.com/reddit/reddit/wiki/API#rules
    """

    e = Extrapolate.Extrapolate()

    def __init__(self, username, password, db_file='local.db',
                 user_agent='Python:storytellingbot:v0.0.0 (by /u/theopakalyse)'):
        """
        Init data layer.
        Login to Reddit. OAuth2 bots gave a rate limit twice as high.
        """
        if DEBUG:
            print('[INFO] Storytellingbot.__init__() called…')
        self.con = sqlite3.connect(db_file)
        self.cur = self.con.cursor()
        self.setup()
        self.reddit = praw.Reddit(user_agent)
        self.reddit.login(username, password)
        if DEBUG:
            print('[INFO] Logged in as', username)

    def __del__(self):
        """
        Remove any locks on the data layer.
        """
        if DEBUG:
            print('[INFO] Storytellingbot.__del__() called…')
        self.con.close()
        if DEBUG:
            print('\tClosed db connection.')

    def setup(self):
        """
        Initialize bot implementation. This only needs to happen the first time
        but the function should be safe to call at any point.
        TODO(@theopak): Create a test subreddit if it does not exist.
        """
        query = '''
            CREATE TABLE IF NOT EXISTS outbox(
                number      integer PRIMARY KEY,-- our unique record identifier
                sent        datetime,           -- 0 if not sent, otherwise timestamp
                keyword     text,               -- keyword that was matched
                id          text,               -- reddit id (hash) of matched comment
                parentId    text,               -- reddit id (hash) of parent comment
                linkUrl     text,               -- permalink of the Submission
                body        text,               -- plaintext body of matched comment
                response    text                -- plaintext body to post as reply
            );'''
        self.cur.execute(query)
        query = '''
            CREATE TABLE IF NOT EXISTS keywords(
                word        text UNIQUE         -- one word or phrase to search for
            );'''
        self.cur.execute(query)
        query = '''
            CREATE TABLE IF NOT EXISTS stories(
                id          integer PRIMARY KEY,-- our unique record identifier
                title       text,               -- plaintext story title
                content     text,               -- plaintext story
                source      text                -- URL
            );'''
        self.cur.execute(query)
        self.con.commit()
        if DEBUG:
            print('[INFO] Setup db')

        # Populate db
        self.cur.execute("SELECT Count() FROM keywords")
        number_of_rows = self.cur.fetchone()[0]
        if number_of_rows == 0:
            sample_keywords = ['unique_keyword_20150416', 'virtualagent']
            self.add_keywords(sample_keywords)
            if DEBUG:
                print('[INFO] Populated db')

    def add_keywords(self, keywords):
        if DEBUG:
            print('[INFO] Storytellingbot.add_keywords(): adding keywords to db:')
        for word in keywords:
            if DEBUG:
                print('\t' + word)
            self.cur.execute('INSERT into keywords VALUES (?)', (word,))
            self.con.commit()
{'fraction_non_alphanumeric': 0.08976398833200742, 'fraction_numerical': 0.007425086184036065, 'mean_word_length': 1.5700170357751277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2711553', 'n_tokens_mistral': 2028, 'n_tokens_neox': 1903, 'n_words': 601}