<jupyter_start><jupyter_text>Imports<jupyter_code>import fire
import sqlite3

from sklearn.feature_extraction.text import CountVectorizer

import nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords

import re
import pandas as pd<jupyter_output>[nltk_data] Downloading package stopwords to /home/anmol/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package punkt to /home/anmol/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package wordnet to /home/anmol/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>Constants<jupyter_code>DEFAULT_DB_RESULTS = '../../reddit-engagement-files/results.db'
FOLLOWED_SUBS = ['codcompetitive']<jupyter_output><empty_output><jupyter_text>Useful Functions<jupyter_code>def preprocess_text(text):
    # remove urls/numbers
    text = str(text).lower()
    text = re.sub(r'http\S+', '', text)
    text = re.sub(r'[0-9]+', '', text)
    # expand contractions
    text = re.sub(r"'ve'", ' have', text)
    text = re.sub(r"n't", ' not', text)
    # remove emojis and special characters
    emoji_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
        u"\U0001F1F2-\U0001F1F4"  # Macau flag
        u"\U0001F1E6-\U0001F1FF"  # flags
        u"\U0001F600-\U0001F64F"
        u"\U00002702-\U000027B0"
        u"\U000024C2-\U0001F251"
        u"\U0001f926-\U0001f937"
        u"\U0001F1F2"
        u"\U0001F1F4"
        u"\U0001F620"
        u"\u200d"
        u"\u2640-\u2642"
        "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)
    text = re.sub('[^a-zA-Z0-9]+', ' ', text)
    # remove stop words
    stops = stopwords.words('english')
    text = ' '.join([x for x in text.split() if x not in stops])
    # tokenize and lemmatize
    text = word_tokenize(text)
    lm = WordNetLemmatizer()
    text = [lm.lemmatize(x) for x in text]
    return text
def read_activity(db_path):
    connection = None
    # exploring ways to ensure file not open elsewhere
    try:
        with open(db_path, 'r') as _:
            connection = sqlite3.connect(db_path)
    except IOError:
        print('In use / other issue accessing %s' % db_path)
    # sort by date and preserve other ordering, return subset
    select_all = ''' SELECT * FROM activity;'''
    df = pd.read_sql_query(select_all, connection)
    df.access_time = pd.to_datetime(df.access_time)
    df = df.reset_index().sort_values(by=['access_time', 'index'])
    df = df[df.subreddit.apply(lambda x: x.lower() not in FOLLOWED_SUBS)]
    df = df.drop(['index'], axis=1)
    return df
def data_from_df(df):
    # combine all X somehow
    # combine all Y somehow
    # countvectorize
    # binary classification
    # f1-score
    pass
# df['new'] = df.title.apply(lambda x: preprocess_text(x))
# df['vote'] = df.action.apply(lambda x: 0 if x=='down' else 1)
# df<jupyter_output><empty_output><jupyter_text>Get Data & Train<jupyter_code>df = read_activity('../../reddit-engagement-files/activity.db')
df.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1348314606741573, 'fraction_numerical': 0.04937906564163217, 'mean_word_length': 3.4338138925294888, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '183450', 'license': 'MIT', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1246, 'n_words': 255, 'path': 'analysis/NLP.ipynb', 'repo_name': 'Anmol-Srivastava/reddit-engagement', 'size': '3382'}