<jupyter_start><jupyter_text>---Import, process, and clean the data------<jupyter_code>import pandas as pd
import numpy as np 

# Load in the data 
#df = pd.read_csv('rspct.tsv', sep="\t")
df = pd.read_csv(r"C:\Users\<NAME>\lambda\sr_predict\reddit_data\rspct.tsv", sep="\t")
df.shape
# Get rid of ambiguous data that can decrese model efficiency 
df = df.drop(['id'], axis=1)

# Combine 'title' and 'selftext' columns to enrich data
df["full_text"] = df["title"] + " " + df["selftext"]
df.head()
# Extract 5 percent of the dataset, even .1 of the data yields memory errors
df = df.sample(frac=0.05, random_state=7)
from bs4 import BeautifulSoup
# Function to clean html tags
def soup(text):
    soup = BeautifulSoup(text, 'html.parser')
    clean_text = soup.get_text()
    return clean_text
df['clean_text'] = df['full_text'].apply(soup)
df.head()
# Taking a look at value count of target 
df['subreddit'].value_counts()
from sklearn.model_selection import train_test_split

# We'll do a 80/20 train/test split and stratify the target 
train, test = train_test_split(df, test_size=0.2, stratify=df["subreddit"])

# Sanity check 
train.shape, test.shape
# Assign data to features/target
X_train = train["clean_text"]
X_test = test["clean_text"]

y_train = train["subreddit"]
y_test = test["subreddit"]

print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)
y_train.head()
# Sanity check v2
y_train.value_counts()<jupyter_output><empty_output><jupyter_text>---Use Label Encoding on the target ------<jupyter_code>from sklearn.preprocessing import LabelEncoder

# Instantiate LabelEncoder
encoder = LabelEncoder()
# Fit the encoder
encoder.fit(y_train)
# Transform on train and test 
y_train = encoder.transform(y_train)
y_test  = encoder.transform(y_test)<jupyter_output><empty_output><jupyter_text>---Create a pipeline equipped with countvectorizer, tfidf-transformer, and a SGDClassifier model------<jupyter_code>from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.pipeline import Pipeline
text_clf = Pipeline([
    ('vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', SGDClassifier()),
    ])
text_clf.fit(X_test, y_test)
test_post = np.array(["I love getting new sneakers; Jordan's, Nike, Addidas, custom footwear. Wanting to see what people think of these fresh kicks. I love the laces, contour of the shoe, more generic sneaker and shoe terms"])
text_clf.predict(test_post)
df.iloc[329][0]
from sklearn.metrics import classification_report

text_clf.fit(X_train,y_train)
predictions = text_clf.predict(X_test)
print(classification_report(y_test,predictions))
from sklearn import metrics
y_train_pred = text_clf.predict(X_train)
print (metrics.accuracy_score(y_train, y_train_pred))<jupyter_output>0.5340078973346496<jupyter_text>---Alright, we can make a prediction and see the top result, lets make a function that returns the top 5 results------<jupyter_code>def get_predictions(post, num_answers=5):
  """ takes a potential post and returns the top options """

  preds = pd.Series(text_clf.decision_function(post)[0])

  preds.index = text_clf.classes_

  preds = preds.sort_values(ascending=False)

  return preds[:5]
get_predictions(test_post)
df.iloc[324]
nba_post = [ """
               LeBron James and Kobe Bryant and both great NBA players and we should stop comparing them.
                """]
get_predictions(nba_post)
df.iloc[738]<jupyter_output><empty_output><jupyter_text>---Testing pipeline with a couple other models really quickly, first RFC------<jupyter_code>from sklearn.ensemble import RandomForestClassifier
text_clf1 = Pipeline([
    ('vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', RandomForestClassifier()),
    ])

text_clf1.fit(X_test, y_test)
test_post = np.array(["I love getting new sneakers; Jordan's, Nike, Addidas, custom footwear. Wanting to see what people think of these fresh kicks"])
text_clf1.predict(test_post)
from sklearn import metrics
y_train_pred1 = text_clf1.predict(X_train)
print (metrics.accuracy_score(y_train, y_train_pred1))<jupyter_output>0.4590819348469891<jupyter_text>---Ok, not optimal, but we got a baseline. Lets pickle the SGD model and try something a little more complex ------<jupyter_code>import pickle

# Save the Model to file in the current working directory
Pkl_Filename = "Baseline_SGD_Model.pkl"  

with open(Pkl_Filename, 'wb') as file:  
    pickle.dump(text_clf, file)<jupyter_output><empty_output><jupyter_text>---The pickled model is too big! Have to break up the pipeline and pickle individually------<jupyter_code># This didn't work, but an attempt was made so we'll leave it in here
#def picklizer(to_pickle, filename, path):
    """
    Creates a pickle file.
    
    Parameters
    ----------
    to_pickle : Python object
        The trained / fitted instance of the 
        transformer or model to be pickled.
    filename : string
        The desired name of the output file,
        not including the '.pkl' extension.
    path : string or path-like object
        The path to the desired output directory.
    """
    import os
    import pickle

    # Create the path to save location
    picklepath = os.path.join(path, filename)

    # Use context manager to open file
    with open(picklepath, "wb") as p:
        pickle.dump(to_pickle, p)
#vect = text_clf.named_steps['vect']
#tfidf = text_clf.named_steps['tfidf']
#clf = text_clf.named_steps['clf']
#filepath = "models/"

# Export vectorizer as pickle
#picklizer(vect, "vec_01.pkl", filepath)

# Export transformer as pickle
#picklizer(tfidf, "tfidf_01.pkl", filepath)

# Export sgd model as pickle
#picklizer(clf, "clf_01.pkl", filepath)<jupyter_output><empty_output><jupyter_text>---Now lets load the pickled model and try to run predictions------<jupyter_code>with open(Pkl_Filename, 'rb') as file:  
    Baseline_SGD_Model = pickle.load(file)

Baseline_SGD_Model
Baseline_SGD_Model.fit(X_test, y_test)
Baseline_SGD_Model.predict(test_post)
from sklearn import metrics
y_train_pred = Baseline_SGD_Model.predict(X_train)
print (metrics.accuracy_score(y_train, y_train_pred))<jupyter_output>0.5342546890424482<jupyter_text>---The only way we could get the pickled model on github was compressing the file with 7zip, lets try to unzip it ------<jupyter_code>pip install py7zr
import py7zr

archive = py7zr.SevenZipFile('Baseline_SGD_Model.7z', mode='r')
archive.extractall(path="/tmp")
archive.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11229135053110774, 'fraction_numerical': 0.014719271623672231, 'mean_word_length': 4.927158273381295, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '135974', 'license': 'MIT', 'n_tokens_mistral': 2321, 'n_tokens_neox': 2220, 'n_words': 603, 'path': 'notebooks/Baseline_TFIDFV_+_SGD_RF.ipynb', 'repo_name': 'tyleretheridge/DS-API', 'size': '6590'}