<jupyter_start><jupyter_text>Data format for Punctuation: Predictions<jupyter_code>import numpy as np
import pandas as pd

import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline

from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
import regex as re

from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import confusion_matrix, plot_confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

import pickle

import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Importing data<jupyter_code>df = pd.read_csv('../../data/clean/anx_writing.csv')
df.tail(2)<jupyter_output><empty_output><jupyter_text>Baseline Accuracy<jupyter_code>df['subreddit'].value_counts(normalize = True)<jupyter_output><empty_output><jupyter_text>Adding stopwords<jupyter_code># Extending my list of stop words
stop_words = stopwords.words("english")
new_stop_words = ['like', 'just', 'don', 've', 'know', 'time', 'really', 
                'want', 'people', 'going', 'think', 'make', 'day', 
                'https', 'com', 'much', 'something', 'would', 'go',
               'even', 'things', 'also', 'got', 'www', 'could', 'take', 'anxiety', 'writer',
                 'ha', 'le', 'wa']
stop_words.extend(new_stop_words)
# Cleaning function
def custom_preprocessor (text):
    text = text.lower() #lowercases word
    text = re.sub(r'[0–9]', '', text) #removes any numbers
    text = re.sub('(<.*?>)', '', text) #removed html
    #copied from https://swatimeena989.medium.com/beginners-guide-for-preprocessing-text-data-f3156bec85ca
    
    lemmatizer = WordNetLemmatizer()
    text = lemmatizer.lemmatize(text)
    
    return text

#copied from https://www.studytonight.com/post/scikitlearn-countvectorizer-in-nlp
# Turn subreddit column to 1 if anxiety and 0 if writing.
df['subreddit'] = df['subreddit'].map({'Anxiety': 'Anxiety', 'writing': 'No Anxiety'})<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code>X = df['text']
y = df['subreddit']
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, train_size=0.7, stratify = y)
pipe = Pipeline([
    ('tvec', TfidfVectorizer(stop_words= stop_words, 
                             preprocessor = custom_preprocessor)),
    ('rf', RandomForestClassifier())  
])
pipe_params = {
    'tvec__max_features': [1000, 2000],
    'tvec__stop_words': [None, 'english'],
    'tvec__ngram_range': [(1,2), (2,2)],
    'rf__n_estimators': [80, 100],
    'rf__max_depth': [None, 1, 3, 5],
    'rf__max_features': ['sqrt', .5]
}
# Instantiate GridSearchCV.
rf_tvec = GridSearchCV(
    pipe,
    param_grid = pipe_params,
    cv = 5,
    n_jobs = -1,
    verbose=1
)
# Fit GridSearch to training data.
rf_tvec.fit(X_train, y_train) 
# Best parameters
rf_tvec.best_params_
# Best parameters
rf_tvec.best_params_
def model_metrics(model, X_train, X_test):
    
    print(f'Best Score: {model.best_score_}')
    print(f'Best Parameters: {model.best_params_}')
    print(f'Train Score: {model.score(X_train, y_train)}')
    print(f'Test Score: {model.score(X_test, y_test)}')
    
model_metrics(rf_tvec, X_train, X_test)<jupyter_output>Best Score: 0.959047619047619
Best Parameters: {'rf__max_depth': None, 'rf__max_features': 'sqrt', 'rf__n_estimators': 100, 'tvec__max_features': 1000, 'tvec__ngram_range': (1, 2), 'tvec__stop_words': 'english'}
Train Score: 0.9976190476190476
Test Score: 0.9461111111111111<jupyter_text>Subreddit Prediction Evaluation ( Anxiety vs Writing )<jupyter_code>#Function for plotting confusion matrix
def plot_matrix(model, X_test):
    
    preds = model.predict(X_test)
    tn, fp, fn, tp = confusion_matrix(y_test, preds).ravel()
    
    fig, ax = plt.subplots(figsize=(7, 7))
    
    # View confusion matrix
    plot_confusion_matrix(model, X_test, y_test, cmap = 'copper', values_format='d', ax = ax)
    
    # Labels, title and ticks
    label_font = {'size':'15'}  # Adjust to fit
    ax.set_xlabel('Predicted labels', fontdict=label_font);
    ax.set_ylabel('Observed labels', fontdict=label_font);

    title_font = {'size':'21'}  # Adjust to fit
    ax.set_title('Confusion Matrix for Random Forest', fontdict=title_font, fontsize=15);

    ax.tick_params(axis='both', which='major', labelsize=12) 
    
#Copied from https://stackoverflow.com/questions/59839782/confusion-matrix-font-size
plot_matrix(rf_tvec, X_test)
# Get predictions
preds_tvec = rf_tvec.predict(X_test)

# Save confusion matrix values
tn, fp, fn, tp = confusion_matrix(y_test, preds_tvec).ravel()

# Calculate the specificity and precision
spec = tn / (tn + fp)
prec = tp / (tp + fp)

print('specificity:', spec)
print('precision:', prec)
pipe.named_steps<jupyter_output><empty_output><jupyter_text>Saving model<jupyter_code>with open('./Random_Forest_punct.pkl', 'wb') as pickle_out:
    pickle_out = pickle.dump(pipe, pickle_out)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13580246913580246, 'fraction_numerical': 0.023319615912208505, 'mean_word_length': 5.076190476190476, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '390898', 'license': 'CC0-1.0', 'n_tokens_mistral': 1930, 'n_tokens_neox': 1814, 'n_words': 385, 'path': 'code/suelem_code/Suelem_3_punct_prediction.ipynb', 'repo_name': 'Ihza430/GA_Project_5', 'size': '5103'}